HMVMXR0.cpp@ 49670

Last change on this file since 49670 was 49670, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: Comment.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 470.7 KB

Line
1	/* $Id: HMVMXR0.cpp 49670 2013-11-26 17:01:01Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	# define HMVMX_SAVE_FULL_GUEST_STATE
40	# define HMVMX_SYNC_FULL_GUEST_STATE
41	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	# define HMVMX_ALWAYS_TRAP_PF
44	# define HMVMX_ALWAYS_SWAP_FPU_STATE
45	# define HMVMX_ALWAYS_FLUSH_TLB
46	#endif
47
48
49	/*******************************************************************************
50	* Defined Constants And Macros *
51	*******************************************************************************/
52	#if defined(RT_ARCH_AMD64)
53	# define HMVMX_IS_64BIT_HOST_MODE() (true)
54	typedef RTHCUINTREG HMVMXHCUINTREG;
55	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
56	extern "C" uint32_t g_fVMXIs64bitHost;
57	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
58	typedef uint64_t HMVMXHCUINTREG;
59	#else
60	# define HMVMX_IS_64BIT_HOST_MODE() (false)
61	typedef RTHCUINTREG HMVMXHCUINTREG;
62	#endif
63
64	/** Use the function table. */
65	#define HMVMX_USE_FUNCTION_TABLE
66
67	/** Determine which tagged-TLB flush handler to use. */
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
70	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
71	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
72
73	/** @name Updated-guest-state flags.
74	* @{ */
75	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
76	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
77	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
78	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
79	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
80	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
81	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
82	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
83	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
84	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
85	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
86	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
91	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
109	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
110	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113	/** @} */
114
115	/** @name
116	* Flags to skip redundant reads of some common VMCS fields that are not part of
117	* the guest-CPU state but are in the transient structure.
118	*/
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
126	/** @} */
127
128	/** @name
129	* States of the VMCS.
130	*
131	* This does not reflect all possible VMCS states but currently only those
132	* needed for maintaining the VMCS consistently even when thread-context hooks
133	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
134	*/
135	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
136	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
137	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
138	/** @} */
139
140	/**
141	* Exception bitmap mask for real-mode guests (real-on-v86).
142	*
143	* We need to intercept all exceptions manually (except #PF). #NM is also
144	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
145	* even in real-mode if we have Nested Paging support.
146	*/
147	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
148	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
149	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
150	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
151	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
153	\| RT_BIT(X86_XCPT_XF))
154
155	/**
156	* Exception bitmap mask for all contributory exceptions.
157	*
158	* Page fault is deliberately excluded here as it's conditional as to whether
159	* it's contributory or benign. Page faults are handled separately.
160	*/
161	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
162	\| RT_BIT(X86_XCPT_DE))
163
164	/** Maximum VM-instruction error number. */
165	#define HMVMX_INSTR_ERROR_MAX 28
166
167	/** Profiling macro. */
168	#ifdef HM_PROFILE_EXIT_DISPATCH
169	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171	#else
172	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174	#endif
175
176	/** Assert that preemption is disabled or covered by thread-context hooks. */
177	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
178	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180	/** Assert that we haven't migrated CPUs when thread-context hooks are not
181	* used. */
182	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184	("Illegal migration! Entered on CPU %u Current %u\n", \
185	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187	/** Helper macro for VM-exit handlers called unexpectedly. */
188	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189	do { \
190	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191	return VERR_VMX_UNEXPECTED_EXIT; \
192	} while (0)
193
194
195	/*******************************************************************************
196	* Structures and Typedefs *
197	*******************************************************************************/
198	/**
199	* VMX transient state.
200	*
201	* A state structure for holding miscellaneous information across
202	* VMX non-root operation and restored after the transition.
203	*/
204	typedef struct VMXTRANSIENT
205	{
206	/** The host's rflags/eflags. */
207	RTCCUINTREG uEflags;
208	#if HC_ARCH_BITS == 32
209	uint32_t u32Alignment0;
210	#endif
211	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
212	uint64_t u64LStarMsr;
213	/** The guest's TPR value used for TPR shadowing. */
214	uint8_t u8GuestTpr;
215	/** Alignment. */
216	uint8_t abAlignment0[7];
217
218	/** The basic VM-exit reason. */
219	uint16_t uExitReason;
220	/** Alignment. */
221	uint16_t u16Alignment0;
222	/** The VM-exit interruption error code. */
223	uint32_t uExitIntErrorCode;
224	/** The VM-exit exit qualification. */
225	uint64_t uExitQualification;
226
227	/** The VM-exit interruption-information field. */
228	uint32_t uExitIntInfo;
229	/** The VM-exit instruction-length field. */
230	uint32_t cbInstr;
231	/** The VM-exit instruction-information field. */
232	union
233	{
234	/** Plain unsigned int representation. */
235	uint32_t u;
236	/** INS and OUTS information. */
237	struct
238	{
239	uint32_t u6Reserved0 : 7;
240	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241	uint32_t u3AddrSize : 3;
242	uint32_t u5Reserved1 : 5;
243	/** The segment register (X86_SREG_XXX). */
244	uint32_t iSegReg : 3;
245	uint32_t uReserved2 : 14;
246	} StrIo;
247	} ExitInstrInfo;
248	/** Whether the VM-entry failed or not. */
249	bool fVMEntryFailed;
250	/** Alignment. */
251	uint8_t abAlignment1[3];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest FPU was active at the time of VM-exit. */
269	bool fWasGuestFPUStateActive;
270	/** Whether the guest debug state was active at the time of VM-exit. */
271	bool fWasGuestDebugStateActive;
272	/** Whether the hyper debug state was active at the time of VM-exit. */
273	bool fWasHyperDebugStateActive;
274	/** Whether TSC-offsetting should be setup before VM-entry. */
275	bool fUpdateTscOffsettingAndPreemptTimer;
276	/** Whether the VM-exit was caused by a page-fault during delivery of a
277	* contributory exception or a page-fault. */
278	bool fVectoringPF;
279	} VMXTRANSIENT;
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
284	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
285	/** Pointer to VMX transient state. */
286	typedef VMXTRANSIENT *PVMXTRANSIENT;
287
288
289	/**
290	* MSR-bitmap read permissions.
291	*/
292	typedef enum VMXMSREXITREAD
293	{
294	/** Reading this MSR causes a VM-exit. */
295	VMXMSREXIT_INTERCEPT_READ = 0xb,
296	/** Reading this MSR does not cause a VM-exit. */
297	VMXMSREXIT_PASSTHRU_READ
298	} VMXMSREXITREAD;
299	/** Pointer to MSR-bitmap read permissions. */
300	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
301
302	/**
303	* MSR-bitmap write permissions.
304	*/
305	typedef enum VMXMSREXITWRITE
306	{
307	/** Writing to this MSR causes a VM-exit. */
308	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
309	/** Writing to this MSR does not cause a VM-exit. */
310	VMXMSREXIT_PASSTHRU_WRITE
311	} VMXMSREXITWRITE;
312	/** Pointer to MSR-bitmap write permissions. */
313	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
314
315
316	/**
317	* VMX VM-exit handler.
318	*
319	* @returns VBox status code.
320	* @param pVCpu Pointer to the VMCPU.
321	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
322	* out-of-sync. Make sure to update the required
323	* fields before using them.
324	* @param pVmxTransient Pointer to the VMX-transient structure.
325	*/
326	#ifndef HMVMX_USE_FUNCTION_TABLE
327	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
328	#else
329	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330	/** Pointer to VM-exit handler. */
331	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
332	#endif
333
334
335	/*******************************************************************************
336	* Internal Functions *
337	*******************************************************************************/
338	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
339	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
340	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
341	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
342	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
343	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
344	#endif
345	#ifndef HMVMX_USE_FUNCTION_TABLE
346	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
347	# define HMVMX_EXIT_DECL static int
348	#else
349	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
350	#endif
351
352	/** @name VM-exit handlers.
353	* @{
354	*/
355	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
356	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
357	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
358	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
359	static FNVMXEXITHANDLER hmR0VmxExitSipi;
360	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
361	static FNVMXEXITHANDLER hmR0VmxExitSmi;
362	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
363	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
364	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
365	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
366	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
367	static FNVMXEXITHANDLER hmR0VmxExitHlt;
368	static FNVMXEXITHANDLER hmR0VmxExitInvd;
369	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
370	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
371	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
372	static FNVMXEXITHANDLER hmR0VmxExitRsm;
373	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
374	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
375	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
376	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
377	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
378	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
379	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
380	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
381	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
382	static FNVMXEXITHANDLER hmR0VmxExitMwait;
383	static FNVMXEXITHANDLER hmR0VmxExitMtf;
384	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
385	static FNVMXEXITHANDLER hmR0VmxExitPause;
386	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
387	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
388	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
391	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
392	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
393	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
394	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
395	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
396	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
397	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
398	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
399	/** @} */
400
401	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
408	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409	#endif
410	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
411
412	/*******************************************************************************
413	* Global Variables *
414	*******************************************************************************/
415	#ifdef HMVMX_USE_FUNCTION_TABLE
416
417	/**
418	* VMX_EXIT dispatch table.
419	*/
420	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
421	{
422	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
423	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
424	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
425	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
426	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
427	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
428	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
429	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
430	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
431	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
432	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
433	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
434	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
435	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
436	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
437	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
438	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
439	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
440	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
441	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
442	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
443	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
444	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
445	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
446	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
447	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
448	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
449	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
450	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
451	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
452	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
453	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
454	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
455	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
456	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
457	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
458	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
459	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
460	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
461	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
462	/* 40 UNDEFINED */ hmR0VmxExitPause,
463	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
464	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
465	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
466	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
467	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
468	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
469	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
470	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
471	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
472	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
473	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
474	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
475	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
476	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
477	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
478	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
479	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
480	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
481	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
482	};
483	#endif /* HMVMX_USE_FUNCTION_TABLE */
484
485	#ifdef VBOX_STRICT
486	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
487	{
488	/* 0 */ "(Not Used)",
489	/* 1 */ "VMCALL executed in VMX root operation.",
490	/* 2 */ "VMCLEAR with invalid physical address.",
491	/* 3 */ "VMCLEAR with VMXON pointer.",
492	/* 4 */ "VMLAUNCH with non-clear VMCS.",
493	/* 5 */ "VMRESUME with non-launched VMCS.",
494	/* 6 */ "VMRESUME after VMXOFF",
495	/* 7 */ "VM entry with invalid control fields.",
496	/* 8 */ "VM entry with invalid host state fields.",
497	/* 9 */ "VMPTRLD with invalid physical address.",
498	/* 10 */ "VMPTRLD with VMXON pointer.",
499	/* 11 */ "VMPTRLD with incorrect revision identifier.",
500	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
501	/* 13 */ "VMWRITE to read-only VMCS component.",
502	/* 14 */ "(Not Used)",
503	/* 15 */ "VMXON executed in VMX root operation.",
504	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
505	/* 17 */ "VM entry with non-launched executing VMCS.",
506	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
507	/* 19 */ "VMCALL with non-clear VMCS.",
508	/* 20 */ "VMCALL with invalid VM-exit control fields.",
509	/* 21 */ "(Not Used)",
510	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
511	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
512	/* 24 */ "VMCALL with invalid SMM-monitor features.",
513	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
514	/* 26 */ "VM entry with events blocked by MOV SS.",
515	/* 27 */ "(Not Used)",
516	/* 28 */ "Invalid operand to INVEPT/INVVPID."
517	};
518	#endif /* VBOX_STRICT */
519
520
521
522	/**
523	* Updates the VM's last error record. If there was a VMX instruction error,
524	* reads the error data from the VMCS and updates VCPU's last error record as
525	* well.
526	*
527	* @param pVM Pointer to the VM.
528	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
529	* VERR_VMX_UNABLE_TO_START_VM or
530	* VERR_VMX_INVALID_VMCS_FIELD).
531	* @param rc The error code.
532	*/
533	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
534	{
535	AssertPtr(pVM);
536	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
537	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
538	{
539	AssertPtrReturnVoid(pVCpu);
540	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
541	}
542	pVM->hm.s.lLastError = rc;
543	}
544
545
546	/**
547	* Reads the VM-entry interruption-information field from the VMCS into the VMX
548	* transient structure.
549	*
550	* @returns VBox status code.
551	* @param pVmxTransient Pointer to the VMX transient structure.
552	*
553	* @remarks No-long-jump zone!!!
554	*/
555	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
556	{
557	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
558	AssertRCReturn(rc, rc);
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Reads the VM-entry exception error code field from the VMCS into
565	* the VMX transient structure.
566	*
567	* @returns VBox status code.
568	* @param pVmxTransient Pointer to the VMX transient structure.
569	*
570	* @remarks No-long-jump zone!!!
571	*/
572	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
573	{
574	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
575	AssertRCReturn(rc, rc);
576	return VINF_SUCCESS;
577	}
578
579
580	/**
581	* Reads the VM-entry exception error code field from the VMCS into
582	* the VMX transient structure.
583	*
584	* @returns VBox status code.
585	* @param pVmxTransient Pointer to the VMX transient structure.
586	*
587	* @remarks No-long-jump zone!!!
588	*/
589	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
590	{
591	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
592	AssertRCReturn(rc, rc);
593	return VINF_SUCCESS;
594	}
595
596
597	/**
598	* Reads the VM-exit interruption-information field from the VMCS into the VMX
599	* transient structure.
600	*
601	* @returns VBox status code.
602	* @param pVmxTransient Pointer to the VMX transient structure.
603	*/
604	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
605	{
606	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
607	{
608	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
609	AssertRCReturn(rc, rc);
610	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
611	}
612	return VINF_SUCCESS;
613	}
614
615
616	/**
617	* Reads the VM-exit interruption error code from the VMCS into the VMX
618	* transient structure.
619	*
620	* @returns VBox status code.
621	* @param pVmxTransient Pointer to the VMX transient structure.
622	*/
623	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
624	{
625	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
626	{
627	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
628	AssertRCReturn(rc, rc);
629	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
630	}
631	return VINF_SUCCESS;
632	}
633
634
635	/**
636	* Reads the VM-exit instruction length field from the VMCS into the VMX
637	* transient structure.
638	*
639	* @returns VBox status code.
640	* @param pVCpu Pointer to the VMCPU.
641	* @param pVmxTransient Pointer to the VMX transient structure.
642	*/
643	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
644	{
645	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
646	{
647	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
648	AssertRCReturn(rc, rc);
649	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
650	}
651	return VINF_SUCCESS;
652	}
653
654
655	/**
656	* Reads the VM-exit instruction-information field from the VMCS into
657	* the VMX transient structure.
658	*
659	* @returns VBox status code.
660	* @param pVmxTransient Pointer to the VMX transient structure.
661	*/
662	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
663	{
664	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
665	{
666	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
667	AssertRCReturn(rc, rc);
668	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
669	}
670	return VINF_SUCCESS;
671	}
672
673
674	/**
675	* Reads the exit qualification from the VMCS into the VMX transient structure.
676	*
677	* @returns VBox status code.
678	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
679	* case).
680	* @param pVmxTransient Pointer to the VMX transient structure.
681	*/
682	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
683	{
684	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
685	{
686	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
687	AssertRCReturn(rc, rc);
688	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
689	}
690	return VINF_SUCCESS;
691	}
692
693
694	/**
695	* Reads the IDT-vectoring information field from the VMCS into the VMX
696	* transient structure.
697	*
698	* @returns VBox status code.
699	* @param pVmxTransient Pointer to the VMX transient structure.
700	*
701	* @remarks No-long-jump zone!!!
702	*/
703	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
704	{
705	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
706	{
707	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
708	AssertRCReturn(rc, rc);
709	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
710	}
711	return VINF_SUCCESS;
712	}
713
714
715	/**
716	* Reads the IDT-vectoring error code from the VMCS into the VMX
717	* transient structure.
718	*
719	* @returns VBox status code.
720	* @param pVmxTransient Pointer to the VMX transient structure.
721	*/
722	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
723	{
724	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
725	{
726	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
727	AssertRCReturn(rc, rc);
728	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
729	}
730	return VINF_SUCCESS;
731	}
732
733
734	/**
735	* Enters VMX root mode operation on the current CPU.
736	*
737	* @returns VBox status code.
738	* @param pVM Pointer to the VM (optional, can be NULL, after
739	* a resume).
740	* @param HCPhysCpuPage Physical address of the VMXON region.
741	* @param pvCpuPage Pointer to the VMXON region.
742	*/
743	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
744	{
745	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
746	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
747	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
748
749	if (pVM)
750	{
751	/* Write the VMCS revision dword to the VMXON region. */
752	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
753	}
754
755	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
756	RTCCUINTREG uEflags = ASMIntDisableFlags();
757
758	/* Enable the VMX bit in CR4 if necessary. */
759	RTCCUINTREG uCr4 = ASMGetCR4();
760	if (!(uCr4 & X86_CR4_VMXE))
761	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
762
763	/* Enter VMX root mode. */
764	int rc = VMXEnable(HCPhysCpuPage);
765	if (RT_FAILURE(rc))
766	ASMSetCR4(uCr4);
767
768	/* Restore interrupts. */
769	ASMSetFlags(uEflags);
770	return rc;
771	}
772
773
774	/**
775	* Exits VMX root mode operation on the current CPU.
776	*
777	* @returns VBox status code.
778	*/
779	static int hmR0VmxLeaveRootMode(void)
780	{
781	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
782
783	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
784	RTCCUINTREG uEflags = ASMIntDisableFlags();
785
786	/* If we're for some reason not in VMX root mode, then don't leave it. */
787	RTCCUINTREG uHostCR4 = ASMGetCR4();
788
789	int rc;
790	if (uHostCR4 & X86_CR4_VMXE)
791	{
792	/* Exit VMX root mode and clear the VMX bit in CR4. */
793	VMXDisable();
794	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
795	rc = VINF_SUCCESS;
796	}
797	else
798	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
799
800	/* Restore interrupts. */
801	ASMSetFlags(uEflags);
802	return rc;
803	}
804
805
806	/**
807	* Allocates and maps one physically contiguous page. The allocated page is
808	* zero'd out. (Used by various VT-x structures).
809	*
810	* @returns IPRT status code.
811	* @param pMemObj Pointer to the ring-0 memory object.
812	* @param ppVirt Where to store the virtual address of the
813	* allocation.
814	* @param pPhys Where to store the physical address of the
815	* allocation.
816	*/
817	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
818	{
819	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
820	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
821	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
822
823	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
824	if (RT_FAILURE(rc))
825	return rc;
826	ppVirt = RTR0MemObjAddress(pMemObj);
827	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
828	ASMMemZero32(*ppVirt, PAGE_SIZE);
829	return VINF_SUCCESS;
830	}
831
832
833	/**
834	* Frees and unmaps an allocated physical page.
835	*
836	* @param pMemObj Pointer to the ring-0 memory object.
837	* @param ppVirt Where to re-initialize the virtual address of
838	* allocation as 0.
839	* @param pHCPhys Where to re-initialize the physical address of the
840	* allocation as 0.
841	*/
842	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
843	{
844	AssertPtr(pMemObj);
845	AssertPtr(ppVirt);
846	AssertPtr(pHCPhys);
847	if (*pMemObj != NIL_RTR0MEMOBJ)
848	{
849	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
850	AssertRC(rc);
851	*pMemObj = NIL_RTR0MEMOBJ;
852	*ppVirt = 0;
853	*pHCPhys = 0;
854	}
855	}
856
857
858	/**
859	* Worker function to free VT-x related structures.
860	*
861	* @returns IPRT status code.
862	* @param pVM Pointer to the VM.
863	*/
864	static void hmR0VmxStructsFree(PVM pVM)
865	{
866	for (VMCPUID i = 0; i < pVM->cCpus; i++)
867	{
868	PVMCPU pVCpu = &pVM->aCpus[i];
869	AssertPtr(pVCpu);
870
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
872	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
873
874	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
875	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
876
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
878	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
879	}
880
881	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
882	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
883	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
884	#endif
885	}
886
887
888	/**
889	* Worker function to allocate VT-x related VM structures.
890	*
891	* @returns IPRT status code.
892	* @param pVM Pointer to the VM.
893	*/
894	static int hmR0VmxStructsAlloc(PVM pVM)
895	{
896	/*
897	* Initialize members up-front so we can cleanup properly on allocation failure.
898	*/
899	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
900	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
901	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
902	pVM->hm.s.vmx.HCPhys##a_Name = 0;
903
904	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
905	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
908
909	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
910	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
911	#endif
912	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
913
914	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
915	for (VMCPUID i = 0; i < pVM->cCpus; i++)
916	{
917	PVMCPU pVCpu = &pVM->aCpus[i];
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
922	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
923	}
924	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
925	#undef VMXLOCAL_INIT_VM_MEMOBJ
926
927	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
928	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
929	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
930	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
931
932	/*
933	* Allocate all the VT-x structures.
934	*/
935	int rc = VINF_SUCCESS;
936	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
937	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
938	if (RT_FAILURE(rc))
939	goto cleanup;
940	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
941	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
942	#endif
943
944	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
945	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
946	{
947	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
948	&pVM->hm.s.vmx.HCPhysApicAccess);
949	if (RT_FAILURE(rc))
950	goto cleanup;
951	}
952
953	/*
954	* Initialize per-VCPU VT-x structures.
955	*/
956	for (VMCPUID i = 0; i < pVM->cCpus; i++)
957	{
958	PVMCPU pVCpu = &pVM->aCpus[i];
959	AssertPtr(pVCpu);
960
961	/* Allocate the VM control structure (VMCS). */
962	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
963	if (RT_FAILURE(rc))
964	goto cleanup;
965
966	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
967	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
968	{
969	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
970	&pVCpu->hm.s.vmx.HCPhysVirtApic);
971	if (RT_FAILURE(rc))
972	goto cleanup;
973	}
974
975	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
976	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
977	{
978	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
979	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
980	if (RT_FAILURE(rc))
981	goto cleanup;
982	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
983	}
984
985	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
986	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
987	if (RT_FAILURE(rc))
988	goto cleanup;
989
990	/* Allocate the VM-exit MSR-load page for the host MSRs. */
991	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
992	if (RT_FAILURE(rc))
993	goto cleanup;
994	}
995
996	return VINF_SUCCESS;
997
998	cleanup:
999	hmR0VmxStructsFree(pVM);
1000	return rc;
1001	}
1002
1003
1004	/**
1005	* Does global VT-x initialization (called during module initialization).
1006	*
1007	* @returns VBox status code.
1008	*/
1009	VMMR0DECL(int) VMXR0GlobalInit(void)
1010	{
1011	#ifdef HMVMX_USE_FUNCTION_TABLE
1012	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1013	# ifdef VBOX_STRICT
1014	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1015	Assert(g_apfnVMExitHandlers[i]);
1016	# endif
1017	#endif
1018	return VINF_SUCCESS;
1019	}
1020
1021
1022	/**
1023	* Does global VT-x termination (called during module termination).
1024	*/
1025	VMMR0DECL(void) VMXR0GlobalTerm()
1026	{
1027	/* Nothing to do currently. */
1028	}
1029
1030
1031	/**
1032	* Sets up and activates VT-x on the current CPU.
1033	*
1034	* @returns VBox status code.
1035	* @param pCpu Pointer to the global CPU info struct.
1036	* @param pVM Pointer to the VM (can be NULL after a host resume
1037	* operation).
1038	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1039	* fEnabledByHost is true).
1040	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1041	* @a fEnabledByHost is true).
1042	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1043	* enable VT-x on the host.
1044	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1045	*/
1046	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1047	void *pvMsrs)
1048	{
1049	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1050	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1051	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1052
1053	/* Enable VT-x if it's not already enabled by the host. */
1054	if (!fEnabledByHost)
1055	{
1056	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1057	if (RT_FAILURE(rc))
1058	return rc;
1059	}
1060
1061	/*
1062	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1063	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1064	*/
1065	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1066	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1067	{
1068	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1069	pCpu->fFlushAsidBeforeUse = false;
1070	}
1071	else
1072	pCpu->fFlushAsidBeforeUse = true;
1073
1074	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1075	++pCpu->cTlbFlushes;
1076
1077	return VINF_SUCCESS;
1078	}
1079
1080
1081	/**
1082	* Deactivates VT-x on the current CPU.
1083	*
1084	* @returns VBox status code.
1085	* @param pCpu Pointer to the global CPU info struct.
1086	* @param pvCpuPage Pointer to the VMXON region.
1087	* @param HCPhysCpuPage Physical address of the VMXON region.
1088	*
1089	* @remarks This function should never be called when SUPR0EnableVTx() or
1090	* similar was used to enable VT-x on the host.
1091	*/
1092	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1093	{
1094	NOREF(pCpu);
1095	NOREF(pvCpuPage);
1096	NOREF(HCPhysCpuPage);
1097
1098	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1099	return hmR0VmxLeaveRootMode();
1100	}
1101
1102
1103	/**
1104	* Sets the permission bits for the specified MSR in the MSR bitmap.
1105	*
1106	* @param pVCpu Pointer to the VMCPU.
1107	* @param uMSR The MSR value.
1108	* @param enmRead Whether reading this MSR causes a VM-exit.
1109	* @param enmWrite Whether writing this MSR causes a VM-exit.
1110	*/
1111	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1112	{
1113	int32_t iBit;
1114	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1115
1116	/*
1117	* Layout:
1118	* 0x000 - 0x3ff - Low MSR read bits
1119	* 0x400 - 0x7ff - High MSR read bits
1120	* 0x800 - 0xbff - Low MSR write bits
1121	* 0xc00 - 0xfff - High MSR write bits
1122	*/
1123	if (uMsr <= 0x00001FFF)
1124	iBit = uMsr;
1125	else if ( uMsr >= 0xC0000000
1126	&& uMsr <= 0xC0001FFF)
1127	{
1128	iBit = (uMsr - 0xC0000000);
1129	pbMsrBitmap += 0x400;
1130	}
1131	else
1132	{
1133	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1134	return;
1135	}
1136
1137	Assert(iBit <= 0x1fff);
1138	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1139	ASMBitSet(pbMsrBitmap, iBit);
1140	else
1141	ASMBitClear(pbMsrBitmap, iBit);
1142
1143	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1144	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1145	else
1146	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1147	}
1148
1149
1150	#ifdef VBOX_STRICT
1151	/**
1152	* Gets the permission bits for the specified MSR in the MSR bitmap.
1153	*
1154	* @returns VBox status code.
1155	* @retval VINF_SUCCESS if the specified MSR is found.
1156	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1157	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1158	*
1159	* @param pVCpu Pointer to the VMCPU.
1160	* @param uMsr The MSR.
1161	* @param penmRead Where to store the read permissions.
1162	* @param penmWrite Where to store the write permissions.
1163	*/
1164	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1165	{
1166	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1167	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1168	int32_t iBit;
1169	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1170
1171	/* See hmR0VmxSetMsrPermission() for the layout. */
1172	if (uMsr <= 0x00001FFF)
1173	iBit = uMsr;
1174	else if ( uMsr >= 0xC0000000
1175	&& uMsr <= 0xC0001FFF)
1176	{
1177	iBit = (uMsr - 0xC0000000);
1178	pbMsrBitmap += 0x400;
1179	}
1180	else
1181	{
1182	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1183	return VERR_NOT_SUPPORTED;
1184	}
1185
1186	Assert(iBit <= 0x1fff);
1187	if (ASMBitTest(pbMsrBitmap, iBit))
1188	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1189	else
1190	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1191
1192	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1193	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1194	else
1195	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1196	return VINF_SUCCESS;
1197	}
1198	#endif /* VBOX_STRICT */
1199
1200
1201	/**
1202	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1203	* area.
1204	*
1205	* @returns VBox status code.
1206	* @param pVCpu Pointer to the VMCPU.
1207	* @param cMsrs The number of MSRs.
1208	*/
1209	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1210	{
1211	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1212	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1213	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1214	{
1215	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1216	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1217	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1218	}
1219
1220	/* Update number of guest MSRs to load/store across the world-switch. */
1221	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1223
1224	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1225	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1226
1227	/* Update the VCPU's copy of the MSR count. */
1228	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1229
1230	return VINF_SUCCESS;
1231	}
1232
1233
1234	/**
1235	* Adds a new (or updates the value of an existing) guest/host MSR
1236	* pair to be swapped during the world-switch as part of the
1237	* auto-load/store MSR area in the VMCS.
1238	*
1239	* @returns VBox status code.
1240	* @param pVCpu Pointer to the VMCPU.
1241	* @param uMsr The MSR.
1242	* @param uGuestMsr Value of the guest MSR.
1243	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1244	* necessary.
1245	*/
1246	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1247	{
1248	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1249	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1250	uint32_t i;
1251	for (i = 0; i < cMsrs; i++)
1252	{
1253	if (pGuestMsr->u32Msr == uMsr)
1254	break;
1255	pGuestMsr++;
1256	}
1257
1258	bool fAdded = false;
1259	if (i == cMsrs)
1260	{
1261	++cMsrs;
1262	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1263	AssertRCReturn(rc, rc);
1264
1265	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1266	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1267	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1268
1269	fAdded = true;
1270	}
1271
1272	/* Update the MSR values in the auto-load/store MSR area. */
1273	pGuestMsr->u32Msr = uMsr;
1274	pGuestMsr->u64Value = uGuestMsrValue;
1275
1276	/* Create/update the MSR slot in the host MSR area. */
1277	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1278	pHostMsr += i;
1279	pHostMsr->u32Msr = uMsr;
1280
1281	/*
1282	* Update the host MSR only when requested by the caller AND when we're
1283	* adding it to the auto-load/store area. Otherwise, it would have been
1284	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1285	*/
1286	if ( fAdded
1287	&& fUpdateHostMsr)
1288	{
1289	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1290	}
1291
1292	return VINF_SUCCESS;
1293	}
1294
1295
1296	/**
1297	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1298	* auto-load/store MSR area in the VMCS.
1299	*
1300	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1301	* area.
1302	*
1303	* @returns VBox status code.
1304	* @param pVCpu Pointer to the VMCPU.
1305	* @param uMsr The MSR.
1306	*/
1307	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1308	{
1309	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1310	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1311	for (uint32_t i = 0; i < cMsrs; i++)
1312	{
1313	/* Find the MSR. */
1314	if (pGuestMsr->u32Msr == uMsr)
1315	{
1316	/* If it's the last MSR, simply reduce the count. */
1317	if (i == cMsrs - 1)
1318	{
1319	--cMsrs;
1320	break;
1321	}
1322
1323	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1324	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1325	pLastGuestMsr += cMsrs;
1326	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1327	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1328
1329	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1330	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1331	pLastHostMsr += cMsrs;
1332	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1333	pHostMsr->u64Value = pLastHostMsr->u64Value;
1334	--cMsrs;
1335	break;
1336	}
1337	pGuestMsr++;
1338	}
1339
1340	/* Update the VMCS if the count changed (meaning the MSR was found). */
1341	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1342	{
1343	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1344	AssertRCReturn(rc, rc);
1345
1346	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1347	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1348	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1349	}
1350
1351	return VINF_SUCCESS;
1352	}
1353
1354
1355	/**
1356	* Checks if the specified guest MSR is part of the auto-load/store area in
1357	* the VMCS.
1358	*
1359	* @returns true if found, false otherwise.
1360	* @param pVCpu Pointer to the VMCPU.
1361	* @param uMsr The MSR to find.
1362	*/
1363	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1364	{
1365	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1366	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1367
1368	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1369	{
1370	if (pGuestMsr->u32Msr == uMsr)
1371	return true;
1372	}
1373	return false;
1374	}
1375
1376
1377	/**
1378	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1379	*
1380	* @param pVCpu Pointer to the VMCPU.
1381	*
1382	* @remarks No-long-jump zone!!!
1383	*/
1384	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1385	{
1386	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1387	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1388	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1389	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1390
1391	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1392	{
1393	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1394	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1395	}
1396
1397	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1398	}
1399
1400
1401	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1402	/**
1403	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1404	* perform lazy restoration of the host MSRs while leaving VT-x.
1405	*
1406	* @param pVCpu Pointer to the VMCPU.
1407	*
1408	* @remarks No-long-jump zone!!!
1409	*/
1410	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1411	{
1412	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1413
1414	#define VMXLOCAL_SAVE_HOST_MSR(uMsr, a_HostMsrField, RestoreFlag) \
1415	do { \
1416	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & (RestoreFlag))) \
1417	{ \
1418	pVCpu->hm.s.vmx.u64Host##a_HostMsrField = ASMRdMsr(uMsr); \
1419	Log4(("hmR0VmxLazySaveHostMsrs: uMsr=%#RX32 HostValue=%#RX64\n", (uMsr), pVCpu->hm.s.vmx.u64Host##a_HostMsrField)); \
1420	} \
1421	} while (0)
1422
1423	/*
1424	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1425	*/
1426	VMXLOCAL_SAVE_HOST_MSR(MSR_K8_LSTAR, LStarMsr, VMX_RESTORE_HOST_MSR_LSTAR);
1427	VMXLOCAL_SAVE_HOST_MSR(MSR_K6_STAR, StarMsr, VMX_RESTORE_HOST_MSR_STAR);
1428	VMXLOCAL_SAVE_HOST_MSR(MSR_K8_SF_MASK, SFMaskMsr, VMX_RESTORE_HOST_MSR_SFMASK);
1429	VMXLOCAL_SAVE_HOST_MSR(MSR_K8_KERNEL_GS_BASE, KernelGSBaseMsr, VMX_RESTORE_HOST_MSR_KERNELGSBASE);
1430	#undef VMXLOCAL_SAVE_HOST_MSR
1431	}
1432
1433
1434	/**
1435	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1436	* lazily while leaving VT-x.
1437	*
1438	* @returns true if it does, false otherwise.
1439	* @param pVCpu Pointer to the VMCPU.
1440	* @param uMsr The MSR to check.
1441	*/
1442	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1443	{
1444	NOREF(pVCpu);
1445	switch (uMsr)
1446	{
1447	case MSR_K8_LSTAR:
1448	case MSR_K6_STAR:
1449	case MSR_K8_SF_MASK:
1450	case MSR_K8_KERNEL_GS_BASE:
1451	return true;
1452	}
1453	return false;
1454	}
1455
1456
1457	/**
1458	* Saves a set of guests MSRs back into the guest-CPU context.
1459	*
1460	* @param pVCpu Pointer to the VMCPU.
1461	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1462	* out-of-sync. Make sure to update the required fields
1463	* before using them.
1464	*
1465	* @remarks No-long-jump zone!!!
1466	*/
1467	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1468	{
1469	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1470	#define VMXLOCAL_SAVE_GUEST_MSR(uMsr, a_GuestMsrField, RestoreFlag) \
1471	do { \
1472	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & (RestoreFlag)) \
1473	{ \
1474	pMixedCtx->msr##a_GuestMsrField = ASMRdMsr(uMsr); \
1475	Log4(("hmR0VmxLazySaveGuestMsrs: uMsr=%#RX32 GuestValue=%#RX64\n", (uMsr), pMixedCtx->msr##a_GuestMsrField)); \
1476	} \
1477	} while (0)
1478
1479	VMXLOCAL_SAVE_GUEST_MSR(MSR_K8_LSTAR, LSTAR, VMX_RESTORE_HOST_MSR_LSTAR);
1480	VMXLOCAL_SAVE_GUEST_MSR(MSR_K6_STAR, STAR, VMX_RESTORE_HOST_MSR_STAR);
1481	VMXLOCAL_SAVE_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, VMX_RESTORE_HOST_MSR_SFMASK);
1482	VMXLOCAL_SAVE_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, VMX_RESTORE_HOST_MSR_KERNELGSBASE);
1483
1484	#undef VMXLOCAL_SAVE_GUEST_MSR
1485	}
1486
1487
1488	/**
1489	* Loads a set of guests MSRs to allow read/passthru to the guest.
1490	*
1491	* The name of this function is slightly confusing. This function does NOT
1492	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1493	* common prefix for functions dealing with "lazy restoration" of the shared
1494	* MSRs.
1495	*
1496	* @param pVCpu Pointer to the VMCPU.
1497	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1498	* out-of-sync. Make sure to update the required fields
1499	* before using them.
1500	*
1501	* @remarks No-long-jump zone!!!
1502	*/
1503	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1504	{
1505	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1506
1507	#define VMXLOCAL_LOAD_GUEST_MSR(uMsr, a_GuestMsrField, a_HostMsrField, RestoreFlag) \
1508	do { \
1509	if ( (pVCpu->hm.s.vmx.fRestoreHostMsrs & (RestoreFlag)) \
1510	\|\| pMixedCtx->msr##a_GuestMsrField != pVCpu->hm.s.vmx.u64Host##a_HostMsrField) \
1511	{ \
1512	ASMWrMsr((uMsr), pMixedCtx->msr##a_GuestMsrField); \
1513	} \
1514	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= (RestoreFlag); \
1515	Log4(("Load: MSRSWAP uMsr=%#RX32 GuestValue=%#RX64\n", (uMsr), pMixedCtx->msr##a_GuestMsrField)); \
1516	} while (0)
1517
1518	VMXLOCAL_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStarMsr, VMX_RESTORE_HOST_MSR_LSTAR);
1519	VMXLOCAL_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, StarMsr, VMX_RESTORE_HOST_MSR_STAR);
1520	VMXLOCAL_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMaskMsr, VMX_RESTORE_HOST_MSR_SFMASK);
1521	VMXLOCAL_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBaseMsr, VMX_RESTORE_HOST_MSR_KERNELGSBASE);
1522
1523	#undef VMXLOCAL_LOAD_GUEST_MSR
1524	}
1525
1526
1527	/**
1528	* Performs lazy restoration of the set of host MSRs if they were previously
1529	* loaded with guest MSR values.
1530	*
1531	* @param pVCpu Pointer to the VMCPU.
1532	*
1533	* @remarks No-long-jump zone!!!
1534	* @remarks The guest MSRs should have been saved back into the guest-CPU
1535	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1536	*/
1537	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1538	{
1539	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1540
1541	#define VMXLOCAL_RESTORE_HOST_MSR(uMsr, a_HostMsrField, RestoreFlag) \
1542	do { \
1543	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & (RestoreFlag)) \
1544	{ \
1545	ASMWrMsr((uMsr), pVCpu->hm.s.vmx.u64Host##a_HostMsrField); \
1546	pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(RestoreFlag); \
1547	Log4(("hmR0VmxLazyRestoreHostMsrs: uMsr=%#RX32 HostValue=%#RX64\n", (uMsr), \
1548	pVCpu->hm.s.vmx.u64Host##a_HostMsrField)); \
1549	} \
1550	} while (0)
1551
1552	VMXLOCAL_RESTORE_HOST_MSR(MSR_K8_LSTAR, LStarMsr, VMX_RESTORE_HOST_MSR_LSTAR);
1553	VMXLOCAL_RESTORE_HOST_MSR(MSR_K6_STAR, StarMsr, VMX_RESTORE_HOST_MSR_STAR);
1554	VMXLOCAL_RESTORE_HOST_MSR(MSR_K8_SF_MASK, SFMaskMsr, VMX_RESTORE_HOST_MSR_SFMASK);
1555	VMXLOCAL_RESTORE_HOST_MSR(MSR_K8_KERNEL_GS_BASE, KernelGSBaseMsr, VMX_RESTORE_HOST_MSR_KERNELGSBASE);
1556
1557	#undef VMXLOCAL_RESTORE_HOST_MSR
1558	}
1559	#endif /* HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
1560
1561
1562	#ifdef VBOX_STRICT
1563	/**
1564	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1565	* VMCS are correct.
1566	*
1567	* @param pVCpu Pointer to the VMCPU.
1568	*/
1569	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1570	{
1571	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1572
1573	/* Verify MSR counts in the VMCS are what we think it should be. */
1574	uint32_t cMsrs;
1575	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1576	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1577
1578	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1579	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1580
1581	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1582	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1583
1584	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1585	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1586	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1587	{
1588	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1589	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1590	pGuestMsr->u32Msr));
1591
1592	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1593	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1594	pHostMsr->u64Value, u64Msr));
1595
1596	/* Verify that the permissions are as expected in the MSR bitmap. */
1597	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1598	{
1599	VMXMSREXITREAD enmRead;
1600	VMXMSREXITWRITE enmWrite;
1601	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1602	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1603	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1604	pGuestMsr->u32Msr));
1605	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1606	pGuestMsr->u32Msr));
1607	}
1608	}
1609	}
1610	# endif /* VBOX_STRICT */
1611
1612
1613	/**
1614	* Flushes the TLB using EPT.
1615	*
1616	* @returns VBox status code.
1617	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1618	* enmFlush).
1619	* @param enmFlush Type of flush.
1620	*
1621	* @remarks Caller is responsible for making sure this function is called only
1622	* when NestedPaging is supported and providing @a enmFlush that is
1623	* supported by the CPU.
1624	* @remarks Can be called with interrupts disabled.
1625	*/
1626	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1627	{
1628	uint64_t au64Descriptor[2];
1629	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1630	au64Descriptor[0] = 0;
1631	else
1632	{
1633	Assert(pVCpu);
1634	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1635	}
1636	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1637
1638	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1639	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1640	rc));
1641	if ( RT_SUCCESS(rc)
1642	&& pVCpu)
1643	{
1644	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1645	}
1646	}
1647
1648
1649	/**
1650	* Flushes the TLB using VPID.
1651	*
1652	* @returns VBox status code.
1653	* @param pVM Pointer to the VM.
1654	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1655	* enmFlush).
1656	* @param enmFlush Type of flush.
1657	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1658	* on @a enmFlush).
1659	*
1660	* @remarks Can be called with interrupts disabled.
1661	*/
1662	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1663	{
1664	NOREF(pVM);
1665	AssertPtr(pVM);
1666	Assert(pVM->hm.s.vmx.fVpid);
1667
1668	uint64_t au64Descriptor[2];
1669	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1670	{
1671	au64Descriptor[0] = 0;
1672	au64Descriptor[1] = 0;
1673	}
1674	else
1675	{
1676	AssertPtr(pVCpu);
1677	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1678	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1679	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1680	au64Descriptor[1] = GCPtr;
1681	}
1682
1683	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1684	AssertMsg(rc == VINF_SUCCESS,
1685	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1686	if ( RT_SUCCESS(rc)
1687	&& pVCpu)
1688	{
1689	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1690	}
1691	}
1692
1693
1694	/**
1695	* Invalidates a guest page by guest virtual address. Only relevant for
1696	* EPT/VPID, otherwise there is nothing really to invalidate.
1697	*
1698	* @returns VBox status code.
1699	* @param pVM Pointer to the VM.
1700	* @param pVCpu Pointer to the VMCPU.
1701	* @param GCVirt Guest virtual address of the page to invalidate.
1702	*/
1703	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1704	{
1705	AssertPtr(pVM);
1706	AssertPtr(pVCpu);
1707	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1708
1709	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1710	if (!fFlushPending)
1711	{
1712	/*
1713	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1714	* See @bugref{6043} and @bugref{6177}.
1715	*
1716	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1717	* function maybe called in a loop with individual addresses.
1718	*/
1719	if (pVM->hm.s.vmx.fVpid)
1720	{
1721	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1722	{
1723	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1724	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1725	}
1726	else
1727	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1728	}
1729	else if (pVM->hm.s.fNestedPaging)
1730	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1731	}
1732
1733	return VINF_SUCCESS;
1734	}
1735
1736
1737	/**
1738	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1739	* otherwise there is nothing really to invalidate.
1740	*
1741	* @returns VBox status code.
1742	* @param pVM Pointer to the VM.
1743	* @param pVCpu Pointer to the VMCPU.
1744	* @param GCPhys Guest physical address of the page to invalidate.
1745	*/
1746	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1747	{
1748	NOREF(pVM); NOREF(GCPhys);
1749	LogFlowFunc(("%RGp\n", GCPhys));
1750
1751	/*
1752	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1753	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1754	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1755	*/
1756	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1757	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1758	return VINF_SUCCESS;
1759	}
1760
1761
1762	/**
1763	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1764	* case where neither EPT nor VPID is supported by the CPU.
1765	*
1766	* @param pVM Pointer to the VM.
1767	* @param pVCpu Pointer to the VMCPU.
1768	* @param pCpu Pointer to the global HM struct.
1769	*
1770	* @remarks Called with interrupts disabled.
1771	*/
1772	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1773	{
1774	AssertPtr(pVCpu);
1775	AssertPtr(pCpu);
1776	NOREF(pVM);
1777
1778	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1779
1780	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1781	#if 0
1782	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1783	pVCpu->hm.s.TlbShootdown.cPages = 0;
1784	#endif
1785
1786	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1787	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1788	pVCpu->hm.s.fForceTLBFlush = false;
1789	return;
1790	}
1791
1792
1793	/**
1794	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1795	*
1796	* @param pVM Pointer to the VM.
1797	* @param pVCpu Pointer to the VMCPU.
1798	* @param pCpu Pointer to the global HM CPU struct.
1799	* @remarks All references to "ASID" in this function pertains to "VPID" in
1800	* Intel's nomenclature. The reason is, to avoid confusion in compare
1801	* statements since the host-CPU copies are named "ASID".
1802	*
1803	* @remarks Called with interrupts disabled.
1804	*/
1805	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1806	{
1807	#ifdef VBOX_WITH_STATISTICS
1808	bool fTlbFlushed = false;
1809	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1810	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1811	if (!fTlbFlushed) \
1812	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1813	} while (0)
1814	#else
1815	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1816	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1817	#endif
1818
1819	AssertPtr(pVM);
1820	AssertPtr(pCpu);
1821	AssertPtr(pVCpu);
1822	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1823	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1824	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1825
1826
1827	/*
1828	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1829	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1830	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1831	*/
1832	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1833	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1834	{
1835	++pCpu->uCurrentAsid;
1836	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1837	{
1838	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1839	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1840	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1841	}
1842
1843	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1844	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1845	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1846
1847	/*
1848	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1849	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1850	*/
1851	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1852	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1853	HMVMX_SET_TAGGED_TLB_FLUSHED();
1854	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1855	}
1856
1857	/* Check for explicit TLB shootdowns. */
1858	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1859	{
1860	/*
1861	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1862	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1863	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1864	* but not guest-physical mappings.
1865	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1866	*/
1867	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1868	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1869	HMVMX_SET_TAGGED_TLB_FLUSHED();
1870	}
1871
1872	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1873	* where it is commented out. Support individual entry flushing
1874	* someday. */
1875	#if 0
1876	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1877	{
1878	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1879
1880	/*
1881	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1882	* as supported by the CPU.
1883	*/
1884	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1885	{
1886	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1887	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1888	}
1889	else
1890	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1891
1892	HMVMX_SET_TAGGED_TLB_FLUSHED();
1893	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1894	pVCpu->hm.s.TlbShootdown.cPages = 0;
1895	}
1896	#endif
1897
1898	pVCpu->hm.s.fForceTLBFlush = false;
1899
1900	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1901
1902	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1903	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1904	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1905	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1906	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1907	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1908	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1909	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1910
1911	/* Update VMCS with the VPID. */
1912	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1913	AssertRC(rc);
1914
1915	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1916	}
1917
1918
1919	/**
1920	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1921	*
1922	* @returns VBox status code.
1923	* @param pVM Pointer to the VM.
1924	* @param pVCpu Pointer to the VMCPU.
1925	* @param pCpu Pointer to the global HM CPU struct.
1926	*
1927	* @remarks Called with interrupts disabled.
1928	*/
1929	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1930	{
1931	AssertPtr(pVM);
1932	AssertPtr(pVCpu);
1933	AssertPtr(pCpu);
1934	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1935	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1936
1937	/*
1938	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1939	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1940	*/
1941	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1942	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1943	{
1944	pVCpu->hm.s.fForceTLBFlush = true;
1945	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1946	}
1947
1948	/* Check for explicit TLB shootdown flushes. */
1949	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1950	{
1951	pVCpu->hm.s.fForceTLBFlush = true;
1952	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1953	}
1954
1955	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1956	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1957
1958	if (pVCpu->hm.s.fForceTLBFlush)
1959	{
1960	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1961	pVCpu->hm.s.fForceTLBFlush = false;
1962	}
1963	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1964	* where it is commented out. Support individual entry flushing
1965	* someday. */
1966	#if 0
1967	else
1968	{
1969	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1970	{
1971	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1972	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1973	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1974	}
1975	else
1976	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1977
1978	pVCpu->hm.s.TlbShootdown.cPages = 0;
1979	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1980	}
1981	#endif
1982	}
1983
1984
1985	/**
1986	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1987	*
1988	* @returns VBox status code.
1989	* @param pVM Pointer to the VM.
1990	* @param pVCpu Pointer to the VMCPU.
1991	* @param pCpu Pointer to the global HM CPU struct.
1992	*
1993	* @remarks Called with interrupts disabled.
1994	*/
1995	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1996	{
1997	AssertPtr(pVM);
1998	AssertPtr(pVCpu);
1999	AssertPtr(pCpu);
2000	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2001	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2002
2003	/*
2004	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2005	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2006	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2007	*/
2008	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2009	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2010	{
2011	pVCpu->hm.s.fForceTLBFlush = true;
2012	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2013	}
2014
2015	/* Check for explicit TLB shootdown flushes. */
2016	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2017	{
2018	/*
2019	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2020	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2021	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2022	*/
2023	pVCpu->hm.s.fForceTLBFlush = true;
2024	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2025	}
2026
2027	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2028	if (pVCpu->hm.s.fForceTLBFlush)
2029	{
2030	++pCpu->uCurrentAsid;
2031	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2032	{
2033	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2034	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2035	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2036	}
2037
2038	pVCpu->hm.s.fForceTLBFlush = false;
2039	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2040	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2041	if (pCpu->fFlushAsidBeforeUse)
2042	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2043	}
2044	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2045	* where it is commented out. Support individual entry flushing
2046	* someday. */
2047	#if 0
2048	else
2049	{
2050	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2051	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2052	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2053	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2054
2055	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2056	{
2057	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2058	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2059	{
2060	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2061	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2062	}
2063	else
2064	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2065
2066	pVCpu->hm.s.TlbShootdown.cPages = 0;
2067	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2068	}
2069	else
2070	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2071	}
2072	#endif
2073
2074	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2075	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2076	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2077	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
2078	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2079	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2080
2081	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2082	AssertRC(rc);
2083	}
2084
2085
2086	/**
2087	* Flushes the guest TLB entry based on CPU capabilities.
2088	*
2089	* @param pVCpu Pointer to the VMCPU.
2090	* @param pCpu Pointer to the global HM CPU struct.
2091	*/
2092	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2093	{
2094	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2095	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2096	#endif
2097	PVM pVM = pVCpu->CTX_SUFF(pVM);
2098	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2099	{
2100	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2101	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2102	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2103	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2104	default:
2105	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2106	break;
2107	}
2108
2109	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2110	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2111
2112	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
2113	}
2114
2115
2116	/**
2117	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2118	* TLB entries from the host TLB before VM-entry.
2119	*
2120	* @returns VBox status code.
2121	* @param pVM Pointer to the VM.
2122	*/
2123	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2124	{
2125	/*
2126	* Determine optimal flush type for Nested Paging.
2127	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2128	* guest execution (see hmR3InitFinalizeR0()).
2129	*/
2130	if (pVM->hm.s.fNestedPaging)
2131	{
2132	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2133	{
2134	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2135	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2136	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2137	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2138	else
2139	{
2140	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2141	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2142	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2143	}
2144
2145	/* Make sure the write-back cacheable memory type for EPT is supported. */
2146	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2147	{
2148	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2149	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2150	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2151	}
2152	}
2153	else
2154	{
2155	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2156	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2157	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2158	}
2159	}
2160
2161	/*
2162	* Determine optimal flush type for VPID.
2163	*/
2164	if (pVM->hm.s.vmx.fVpid)
2165	{
2166	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2167	{
2168	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2169	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2170	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2171	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2172	else
2173	{
2174	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2175	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2176	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2177	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2178	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2179	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2180	pVM->hm.s.vmx.fVpid = false;
2181	}
2182	}
2183	else
2184	{
2185	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2186	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2187	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2188	pVM->hm.s.vmx.fVpid = false;
2189	}
2190	}
2191
2192	/*
2193	* Setup the handler for flushing tagged-TLBs.
2194	*/
2195	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2196	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2197	else if (pVM->hm.s.fNestedPaging)
2198	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2199	else if (pVM->hm.s.vmx.fVpid)
2200	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2201	else
2202	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2203	return VINF_SUCCESS;
2204	}
2205
2206
2207	/**
2208	* Sets up pin-based VM-execution controls in the VMCS.
2209	*
2210	* @returns VBox status code.
2211	* @param pVM Pointer to the VM.
2212	* @param pVCpu Pointer to the VMCPU.
2213	*/
2214	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2215	{
2216	AssertPtr(pVM);
2217	AssertPtr(pVCpu);
2218
2219	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2220	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2221
2222	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2223	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2224	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2225
2226	/* Enable the VMX preemption timer. */
2227	if (pVM->hm.s.vmx.fUsePreemptTimer)
2228	{
2229	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2230	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2231	}
2232
2233	if ((val & zap) != val)
2234	{
2235	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2236	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2237	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2238	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2239	}
2240
2241	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2242	AssertRCReturn(rc, rc);
2243
2244	/* Update VCPU with the currently set pin-based VM-execution controls. */
2245	pVCpu->hm.s.vmx.u32PinCtls = val;
2246	return rc;
2247	}
2248
2249
2250	/**
2251	* Sets up processor-based VM-execution controls in the VMCS.
2252	*
2253	* @returns VBox status code.
2254	* @param pVM Pointer to the VM.
2255	* @param pVMCPU Pointer to the VMCPU.
2256	*/
2257	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2258	{
2259	AssertPtr(pVM);
2260	AssertPtr(pVCpu);
2261
2262	int rc = VERR_INTERNAL_ERROR_5;
2263	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2264	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2265
2266	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2267	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2268	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2269	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2270	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2271	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2272	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2273
2274	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2275	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2276	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2277	{
2278	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2279	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2280	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2281	}
2282
2283	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2284	if (!pVM->hm.s.fNestedPaging)
2285	{
2286	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2287	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2288	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2289	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2290	}
2291
2292	/* Use TPR shadowing if supported by the CPU. */
2293	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2294	{
2295	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2296	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2297	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2298	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2299	AssertRCReturn(rc, rc);
2300
2301	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2302	/* CR8 writes causes a VM-exit based on TPR threshold. */
2303	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2304	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2305	}
2306	else
2307	{
2308	/*
2309	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2310	* Set this control only for 64-bit guests.
2311	*/
2312	if (pVM->hm.s.fAllow64BitGuests)
2313	{
2314	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2315	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2316	}
2317	}
2318
2319	/* Use MSR-bitmaps if supported by the CPU. */
2320	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2321	{
2322	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2323
2324	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2325	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2326	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2327	AssertRCReturn(rc, rc);
2328
2329	/*
2330	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2331	* automatically as dedicated fields in the VMCS.
2332	*/
2333	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2334	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2335	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2336	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2337	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2338
2339	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2340	/*
2341	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2342	*/
2343	if ( HMVMX_IS_64BIT_HOST_MODE()
2344	&& pVM->hm.s.fAllow64BitGuests)
2345	{
2346	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2347	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2348	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2349	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2350	}
2351	#endif
2352	}
2353
2354	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2355	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2356	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2357
2358	if ((val & zap) != val)
2359	{
2360	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2361	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2362	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2363	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2364	}
2365
2366	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2367	AssertRCReturn(rc, rc);
2368
2369	/* Update VCPU with the currently set processor-based VM-execution controls. */
2370	pVCpu->hm.s.vmx.u32ProcCtls = val;
2371
2372	/*
2373	* Secondary processor-based VM-execution controls.
2374	*/
2375	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2376	{
2377	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2378	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2379
2380	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2381	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2382
2383	if (pVM->hm.s.fNestedPaging)
2384	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2385	else
2386	{
2387	/*
2388	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2389	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2390	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2391	*/
2392	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2393	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2394	}
2395
2396	if (pVM->hm.s.vmx.fVpid)
2397	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2398
2399	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2400	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2401
2402	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2403	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2404	* done dynamically. */
2405	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2406	{
2407	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2408	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2409	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2410	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2411	AssertRCReturn(rc, rc);
2412	}
2413
2414	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2415	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2416
2417	if ((val & zap) != val)
2418	{
2419	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2420	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2421	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2422	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2423	}
2424
2425	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2426	AssertRCReturn(rc, rc);
2427
2428	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2429	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2430	}
2431	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2432	{
2433	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2434	"available\n"));
2435	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2436	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2437	}
2438
2439	return VINF_SUCCESS;
2440	}
2441
2442
2443	/**
2444	* Sets up miscellaneous (everything other than Pin & Processor-based
2445	* VM-execution) control fields in the VMCS.
2446	*
2447	* @returns VBox status code.
2448	* @param pVM Pointer to the VM.
2449	* @param pVCpu Pointer to the VMCPU.
2450	*/
2451	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2452	{
2453	NOREF(pVM);
2454	AssertPtr(pVM);
2455	AssertPtr(pVCpu);
2456
2457	int rc = VERR_GENERAL_FAILURE;
2458
2459	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2460	#if 0
2461	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2462	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2463	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2464
2465	/*
2466	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2467	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2468	* We thus use the exception bitmap to control it rather than use both.
2469	*/
2470	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2471	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2472
2473	/** @todo Explore possibility of using IO-bitmaps. */
2474	/* All IO & IOIO instructions cause VM-exits. */
2475	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2476	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2477
2478	/* Initialize the MSR-bitmap area. */
2479	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2480	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2481	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2482	#endif
2483
2484	/* Setup MSR auto-load/store area. */
2485	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2486	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2487	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2488	AssertRCReturn(rc, rc);
2489	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2490	AssertRCReturn(rc, rc);
2491
2492	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2493	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2494	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2495	AssertRCReturn(rc, rc);
2496
2497	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2498	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2499	AssertRCReturn(rc, rc);
2500
2501	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2502	#if 0
2503	/* Setup debug controls */
2504	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2505	AssertRCReturn(rc, rc);
2506	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2507	AssertRCReturn(rc, rc);
2508	#endif
2509
2510	return rc;
2511	}
2512
2513
2514	/**
2515	* Sets up the initial exception bitmap in the VMCS based on static conditions
2516	* (i.e. conditions that cannot ever change after starting the VM).
2517	*
2518	* @returns VBox status code.
2519	* @param pVM Pointer to the VM.
2520	* @param pVCpu Pointer to the VMCPU.
2521	*/
2522	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2523	{
2524	AssertPtr(pVM);
2525	AssertPtr(pVCpu);
2526
2527	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2528
2529	uint32_t u32XcptBitmap = 0;
2530
2531	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2532	if (!pVM->hm.s.fNestedPaging)
2533	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2534
2535	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2536	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2537	AssertRCReturn(rc, rc);
2538	return rc;
2539	}
2540
2541
2542	/**
2543	* Sets up the initial guest-state mask. The guest-state mask is consulted
2544	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2545	* for the nested virtualization case (as it would cause a VM-exit).
2546	*
2547	* @param pVCpu Pointer to the VMCPU.
2548	*/
2549	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2550	{
2551	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2552	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2553	return VINF_SUCCESS;
2554	}
2555
2556
2557	/**
2558	* Does per-VM VT-x initialization.
2559	*
2560	* @returns VBox status code.
2561	* @param pVM Pointer to the VM.
2562	*/
2563	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2564	{
2565	LogFlowFunc(("pVM=%p\n", pVM));
2566
2567	int rc = hmR0VmxStructsAlloc(pVM);
2568	if (RT_FAILURE(rc))
2569	{
2570	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2571	return rc;
2572	}
2573
2574	return VINF_SUCCESS;
2575	}
2576
2577
2578	/**
2579	* Does per-VM VT-x termination.
2580	*
2581	* @returns VBox status code.
2582	* @param pVM Pointer to the VM.
2583	*/
2584	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2585	{
2586	LogFlowFunc(("pVM=%p\n", pVM));
2587
2588	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2589	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2590	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2591	#endif
2592	hmR0VmxStructsFree(pVM);
2593	return VINF_SUCCESS;
2594	}
2595
2596
2597	/**
2598	* Sets up the VM for execution under VT-x.
2599	* This function is only called once per-VM during initialization.
2600	*
2601	* @returns VBox status code.
2602	* @param pVM Pointer to the VM.
2603	*/
2604	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2605	{
2606	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2607	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2608
2609	LogFlowFunc(("pVM=%p\n", pVM));
2610
2611	/*
2612	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2613	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2614	*/
2615	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2616	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2617	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2618	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2619	{
2620	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2621	return VERR_INTERNAL_ERROR;
2622	}
2623
2624	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2625	/*
2626	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2627	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2628	*/
2629	if ( pVM->hm.s.fAllow64BitGuests
2630	&& !HMVMX_IS_64BIT_HOST_MODE())
2631	{
2632	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2633	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2634	}
2635	#endif
2636
2637	/* Initialize these always, see hmR3InitFinalizeR0().*/
2638	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2639	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2640
2641	/* Setup the tagged-TLB flush handlers. */
2642	int rc = hmR0VmxSetupTaggedTlb(pVM);
2643	if (RT_FAILURE(rc))
2644	{
2645	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2646	return rc;
2647	}
2648
2649	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2650	{
2651	PVMCPU pVCpu = &pVM->aCpus[i];
2652	AssertPtr(pVCpu);
2653	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2654
2655	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2656	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2657
2658	/* Set revision dword at the beginning of the VMCS structure. */
2659	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2660
2661	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2662	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2663	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2664	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2665
2666	/* Load this VMCS as the current VMCS. */
2667	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2668	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2669	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2670
2671	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2672	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2673	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2674
2675	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2676	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2677	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2678
2679	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2680	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2681	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2682
2683	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2684	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2685	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2686
2687	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2688	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2689	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2690
2691	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2692	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2693	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2694	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2695	#endif
2696
2697	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2698	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2699	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2700	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2701
2702	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2703
2704	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2705	}
2706
2707	return VINF_SUCCESS;
2708	}
2709
2710
2711	/**
2712	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2713	* the VMCS.
2714	*
2715	* @returns VBox status code.
2716	* @param pVM Pointer to the VM.
2717	* @param pVCpu Pointer to the VMCPU.
2718	*/
2719	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2720	{
2721	NOREF(pVM); NOREF(pVCpu);
2722
2723	RTCCUINTREG uReg = ASMGetCR0();
2724	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2725	AssertRCReturn(rc, rc);
2726
2727	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2728	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2729	if (HMVMX_IS_64BIT_HOST_MODE())
2730	{
2731	uint64_t uRegCR3 = HMR0Get64bitCR3();
2732	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2733	}
2734	else
2735	#endif
2736	{
2737	uReg = ASMGetCR3();
2738	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2739	}
2740	AssertRCReturn(rc, rc);
2741
2742	uReg = ASMGetCR4();
2743	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2744	AssertRCReturn(rc, rc);
2745	return rc;
2746	}
2747
2748
2749	#if HC_ARCH_BITS == 64
2750	/**
2751	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2752	* requirements. See hmR0VmxSaveHostSegmentRegs().
2753	*/
2754	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2755	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2756	{ \
2757	bool fValidSelector = true; \
2758	if ((selValue) & X86_SEL_LDT) \
2759	{ \
2760	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2761	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2762	} \
2763	if (fValidSelector) \
2764	{ \
2765	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2766	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2767	} \
2768	(selValue) = 0; \
2769	}
2770	#endif
2771
2772
2773	/**
2774	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2775	* the host-state area in the VMCS.
2776	*
2777	* @returns VBox status code.
2778	* @param pVM Pointer to the VM.
2779	* @param pVCpu Pointer to the VMCPU.
2780	*/
2781	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2782	{
2783	NOREF(pVM);
2784	int rc = VERR_INTERNAL_ERROR_5;
2785
2786	/*
2787	* Host DS, ES, FS and GS segment registers.
2788	*/
2789	#if HC_ARCH_BITS == 64
2790	RTSEL uSelDS = ASMGetDS();
2791	RTSEL uSelES = ASMGetES();
2792	RTSEL uSelFS = ASMGetFS();
2793	RTSEL uSelGS = ASMGetGS();
2794	#else
2795	RTSEL uSelDS = 0;
2796	RTSEL uSelES = 0;
2797	RTSEL uSelFS = 0;
2798	RTSEL uSelGS = 0;
2799	#endif
2800
2801	/* Recalculate which host-state bits need to be manually restored. */
2802	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2803
2804	/*
2805	* Host CS and SS segment registers.
2806	*/
2807	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2808	RTSEL uSelCS;
2809	RTSEL uSelSS;
2810	if (HMVMX_IS_64BIT_HOST_MODE())
2811	{
2812	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2813	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2814	}
2815	else
2816	{
2817	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2818	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2819	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2820	}
2821	#else
2822	RTSEL uSelCS = ASMGetCS();
2823	RTSEL uSelSS = ASMGetSS();
2824	#endif
2825
2826	/*
2827	* Host TR segment register.
2828	*/
2829	RTSEL uSelTR = ASMGetTR();
2830
2831	#if HC_ARCH_BITS == 64
2832	/*
2833	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2834	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2835	*/
2836	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2837	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2838	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2839	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2840	# undef VMXLOCAL_ADJUST_HOST_SEG
2841	#endif
2842
2843	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2844	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2845	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2846	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2847	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2848	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2849	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2850	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2851	Assert(uSelCS);
2852	Assert(uSelTR);
2853
2854	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2855	#if 0
2856	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2857	Assert(uSelSS != 0);
2858	#endif
2859
2860	/* Write these host selector fields into the host-state area in the VMCS. */
2861	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2862	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2863	#if HC_ARCH_BITS == 64
2864	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2865	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2866	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2867	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2868	#endif
2869	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2870
2871	/*
2872	* Host GDTR and IDTR.
2873	*/
2874	RTGDTR Gdtr;
2875	RT_ZERO(Gdtr);
2876	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2877	if (HMVMX_IS_64BIT_HOST_MODE())
2878	{
2879	X86XDTR64 Gdtr64;
2880	X86XDTR64 Idtr64;
2881	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2882	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2883	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2884
2885	Gdtr.cbGdt = Gdtr64.cb;
2886	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2887	}
2888	else
2889	#endif
2890	{
2891	RTIDTR Idtr;
2892	ASMGetGDTR(&Gdtr);
2893	ASMGetIDTR(&Idtr);
2894	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2895	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2896
2897	#if HC_ARCH_BITS == 64
2898	/*
2899	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2900	* maximum limit (0xffff) on every VM-exit.
2901	*/
2902	if (Gdtr.cbGdt != 0xffff)
2903	{
2904	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2905	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2906	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2907	}
2908
2909	/*
2910	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2911	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2912	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2913	*/
2914	if (Idtr.cbIdt < 0x0fff)
2915	{
2916	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2917	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2918	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2919	}
2920	#endif
2921	}
2922
2923	/*
2924	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2925	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2926	*/
2927	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2928	{
2929	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2930	return VERR_VMX_INVALID_HOST_STATE;
2931	}
2932
2933	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2934	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2935	if (HMVMX_IS_64BIT_HOST_MODE())
2936	{
2937	/* We need the 64-bit TR base for hybrid darwin. */
2938	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2939	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2940	}
2941	else
2942	#endif
2943	{
2944	uintptr_t uTRBase;
2945	#if HC_ARCH_BITS == 64
2946	uTRBase = X86DESC64_BASE(pDesc);
2947
2948	/*
2949	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2950	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2951	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2952	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2953	*
2954	* [1] See Intel spec. 3.5 "System Descriptor Types".
2955	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2956	*/
2957	Assert(pDesc->System.u4Type == 11);
2958	if ( pDesc->System.u16LimitLow != 0x67
2959	\|\| pDesc->System.u4LimitHigh)
2960	{
2961	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2962	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2963
2964	/* Store the GDTR here as we need it while restoring TR. */
2965	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2966	}
2967	#else
2968	uTRBase = X86DESC_BASE(pDesc);
2969	#endif
2970	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2971	}
2972	AssertRCReturn(rc, rc);
2973
2974	/*
2975	* Host FS base and GS base.
2976	*/
2977	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2978	if (HMVMX_IS_64BIT_HOST_MODE())
2979	{
2980	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2981	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2982	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2983	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2984
2985	# if HC_ARCH_BITS == 64
2986	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2987	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2988	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2989	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2990	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2991	# endif
2992	}
2993	#endif
2994	return rc;
2995	}
2996
2997
2998	/**
2999	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3000	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3001	* the host after every successful VM exit.
3002	*
3003	* @returns VBox status code.
3004	* @param pVM Pointer to the VM.
3005	* @param pVCpu Pointer to the VMCPU.
3006	*
3007	* @remarks No-long-jump zone!!!
3008	*/
3009	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3010	{
3011	NOREF(pVM);
3012
3013	AssertPtr(pVCpu);
3014	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3015
3016	int rc = VINF_SUCCESS;
3017	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3018	if ( HMVMX_IS_64BIT_HOST_MODE()
3019	&& pVM->hm.s.fAllow64BitGuests)
3020	{
3021	hmR0VmxLazySaveHostMsrs(pVCpu);
3022	}
3023	#endif
3024
3025	if (pVCpu->hm.s.vmx.cMsrs > 0)
3026	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
3027
3028	/*
3029	* Host Sysenter MSRs.
3030	*/
3031	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3032	AssertRCReturn(rc, rc);
3033	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3034	if (HMVMX_IS_64BIT_HOST_MODE())
3035	{
3036	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3037	AssertRCReturn(rc, rc);
3038	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3039	}
3040	else
3041	{
3042	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3043	AssertRCReturn(rc, rc);
3044	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3045	}
3046	#elif HC_ARCH_BITS == 32
3047	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3048	AssertRCReturn(rc, rc);
3049	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3050	#else
3051	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3052	AssertRCReturn(rc, rc);
3053	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3054	#endif
3055	AssertRCReturn(rc, rc);
3056
3057	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3058	* hmR0VmxSetupExitCtls() !! */
3059	return rc;
3060	}
3061
3062
3063	/**
3064	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3065	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3066	* controls".
3067	*
3068	* @returns VBox status code.
3069	* @param pVCpu Pointer to the VMCPU.
3070	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3071	* out-of-sync. Make sure to update the required fields
3072	* before using them.
3073	*
3074	* @remarks No-long-jump zone!!!
3075	*/
3076	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3077	{
3078	int rc = VINF_SUCCESS;
3079	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3080	{
3081	PVM pVM = pVCpu->CTX_SUFF(pVM);
3082	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3083	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3084
3085	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3086	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3087
3088	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3089	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3090	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3091	else
3092	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3093
3094	/*
3095	* The following should -not- be set (since we're not in SMM mode):
3096	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3097	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3098	*/
3099
3100	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3101	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3102	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3103
3104	if ((val & zap) != val)
3105	{
3106	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3107	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3108	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3109	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3110	}
3111
3112	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3113	AssertRCReturn(rc, rc);
3114
3115	/* Update VCPU with the currently set VM-exit controls. */
3116	pVCpu->hm.s.vmx.u32EntryCtls = val;
3117	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3118	}
3119	return rc;
3120	}
3121
3122
3123	/**
3124	* Sets up the VM-exit controls in the VMCS.
3125	*
3126	* @returns VBox status code.
3127	* @param pVM Pointer to the VM.
3128	* @param pVCpu Pointer to the VMCPU.
3129	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3130	* out-of-sync. Make sure to update the required fields
3131	* before using them.
3132	*
3133	* @remarks requires EFER.
3134	*/
3135	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3136	{
3137	NOREF(pMixedCtx);
3138
3139	int rc = VINF_SUCCESS;
3140	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3141	{
3142	PVM pVM = pVCpu->CTX_SUFF(pVM);
3143	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3144	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3145
3146	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3147	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3148
3149	/*
3150	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3151	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3152	*/
3153	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3154	if (HMVMX_IS_64BIT_HOST_MODE())
3155	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3156	else
3157	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3158	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3159	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3160	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3161	else
3162	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3163	#endif
3164
3165	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3166	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3167
3168	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3169	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3170	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3171	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3172	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3173
3174	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3175	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3176
3177	if ((val & zap) != val)
3178	{
3179	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3180	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3181	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3182	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3183	}
3184
3185	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3186	AssertRCReturn(rc, rc);
3187
3188	/* Update VCPU with the currently set VM-exit controls. */
3189	pVCpu->hm.s.vmx.u32ExitCtls = val;
3190	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3191	}
3192	return rc;
3193	}
3194
3195
3196	/**
3197	* Loads the guest APIC and related state.
3198	*
3199	* @returns VBox status code.
3200	* @param pVM Pointer to the VM.
3201	* @param pVCpu Pointer to the VMCPU.
3202	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3203	* out-of-sync. Make sure to update the required fields
3204	* before using them.
3205	*/
3206	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3207	{
3208	NOREF(pMixedCtx);
3209
3210	int rc = VINF_SUCCESS;
3211	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3212	{
3213	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3214	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3215	{
3216	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3217
3218	bool fPendingIntr = false;
3219	uint8_t u8Tpr = 0;
3220	uint8_t u8PendingIntr = 0;
3221	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3222	AssertRCReturn(rc, rc);
3223
3224	/*
3225	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3226	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3227	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3228	* the interrupt when we VM-exit for other reasons.
3229	*/
3230	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3231	uint32_t u32TprThreshold = 0;
3232	if (fPendingIntr)
3233	{
3234	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3235	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3236	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3237	if (u8PendingPriority <= u8TprPriority)
3238	u32TprThreshold = u8PendingPriority;
3239	else
3240	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3241	}
3242	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3243
3244	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3245	AssertRCReturn(rc, rc);
3246	}
3247
3248	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3249	}
3250	return rc;
3251	}
3252
3253
3254	/**
3255	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3256	*
3257	* @returns Guest's interruptibility-state.
3258	* @param pVCpu Pointer to the VMCPU.
3259	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3260	* out-of-sync. Make sure to update the required fields
3261	* before using them.
3262	*
3263	* @remarks No-long-jump zone!!!
3264	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3265	*/
3266	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3267	{
3268	/*
3269	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3270	* inhibit interrupts or clear any existing interrupt-inhibition.
3271	*/
3272	uint32_t uIntrState = 0;
3273	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3274	{
3275	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3276	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
3277	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
3278	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3279	{
3280	/*
3281	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3282	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3283	*/
3284	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3285	}
3286	else if (pMixedCtx->eflags.Bits.u1IF)
3287	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3288	else
3289	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3290	}
3291	return uIntrState;
3292	}
3293
3294
3295	/**
3296	* Loads the guest's interruptibility-state into the guest-state area in the
3297	* VMCS.
3298	*
3299	* @returns VBox status code.
3300	* @param pVCpu Pointer to the VMCPU.
3301	* @param uIntrState The interruptibility-state to set.
3302	*/
3303	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3304	{
3305	NOREF(pVCpu);
3306	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3307	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3308	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3309	AssertRCReturn(rc, rc);
3310	return rc;
3311	}
3312
3313
3314	/**
3315	* Loads the guest's RIP into the guest-state area in the VMCS.
3316	*
3317	* @returns VBox status code.
3318	* @param pVCpu Pointer to the VMCPU.
3319	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3320	* out-of-sync. Make sure to update the required fields
3321	* before using them.
3322	*
3323	* @remarks No-long-jump zone!!!
3324	*/
3325	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3326	{
3327	int rc = VINF_SUCCESS;
3328	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3329	{
3330	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3331	AssertRCReturn(rc, rc);
3332
3333	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3334	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
3335	}
3336	return rc;
3337	}
3338
3339
3340	/**
3341	* Loads the guest's RSP into the guest-state area in the VMCS.
3342	*
3343	* @returns VBox status code.
3344	* @param pVCpu Pointer to the VMCPU.
3345	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3346	* out-of-sync. Make sure to update the required fields
3347	* before using them.
3348	*
3349	* @remarks No-long-jump zone!!!
3350	*/
3351	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3352	{
3353	int rc = VINF_SUCCESS;
3354	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3355	{
3356	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3357	AssertRCReturn(rc, rc);
3358
3359	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3360	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3361	}
3362	return rc;
3363	}
3364
3365
3366	/**
3367	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3368	*
3369	* @returns VBox status code.
3370	* @param pVCpu Pointer to the VMCPU.
3371	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3372	* out-of-sync. Make sure to update the required fields
3373	* before using them.
3374	*
3375	* @remarks No-long-jump zone!!!
3376	*/
3377	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3378	{
3379	int rc = VINF_SUCCESS;
3380	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3381	{
3382	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3383	Let us assert it as such and use 32-bit VMWRITE. */
3384	Assert(!(pMixedCtx->rflags.u64 >> 32));
3385	X86EFLAGS Eflags = pMixedCtx->eflags;
3386	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3387	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3388
3389	/*
3390	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
3391	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3392	*/
3393	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3394	{
3395	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3396	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3397	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3398	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3399	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3400	}
3401
3402	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3403	AssertRCReturn(rc, rc);
3404
3405	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3406	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3407	}
3408	return rc;
3409	}
3410
3411
3412	/**
3413	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3414	*
3415	* @returns VBox status code.
3416	* @param pVCpu Pointer to the VMCPU.
3417	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3418	* out-of-sync. Make sure to update the required fields
3419	* before using them.
3420	*
3421	* @remarks No-long-jump zone!!!
3422	*/
3423	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3424	{
3425	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3426	AssertRCReturn(rc, rc);
3427	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3428	AssertRCReturn(rc, rc);
3429	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3430	AssertRCReturn(rc, rc);
3431	return rc;
3432	}
3433
3434
3435	/**
3436	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3437	* CR0 is partially shared with the host and we have to consider the FPU bits.
3438	*
3439	* @returns VBox status code.
3440	* @param pVM Pointer to the VM.
3441	* @param pVCpu Pointer to the VMCPU.
3442	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3443	* out-of-sync. Make sure to update the required fields
3444	* before using them.
3445	*
3446	* @remarks No-long-jump zone!!!
3447	*/
3448	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3449	{
3450	/*
3451	* Guest CR0.
3452	* Guest FPU.
3453	*/
3454	int rc = VINF_SUCCESS;
3455	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3456	{
3457	Assert(!(pMixedCtx->cr0 >> 32));
3458	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3459	PVM pVM = pVCpu->CTX_SUFF(pVM);
3460
3461	/* The guest's view (read access) of its CR0 is unblemished. */
3462	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3463	AssertRCReturn(rc, rc);
3464	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3465
3466	/* Setup VT-x's view of the guest CR0. */
3467	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3468	if (pVM->hm.s.fNestedPaging)
3469	{
3470	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3471	{
3472	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3473	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3474	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3475	}
3476	else
3477	{
3478	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3479	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3480	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3481	}
3482
3483	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3484	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3485	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3486
3487	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3488	AssertRCReturn(rc, rc);
3489	}
3490	else
3491	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3492
3493	/*
3494	* Guest FPU bits.
3495	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3496	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3497	*/
3498	u32GuestCR0 \|= X86_CR0_NE;
3499	bool fInterceptNM = false;
3500	if (CPUMIsGuestFPUStateActive(pVCpu))
3501	{
3502	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3503	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3504	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3505	}
3506	else
3507	{
3508	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3509	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3510	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3511	}
3512
3513	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3514	bool fInterceptMF = false;
3515	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3516	fInterceptMF = true;
3517
3518	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3519	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3520	{
3521	Assert(PDMVmmDevHeapIsEnabled(pVM));
3522	Assert(pVM->hm.s.vmx.pRealModeTSS);
3523	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3524	fInterceptNM = true;
3525	fInterceptMF = true;
3526	}
3527	else
3528	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3529
3530	if (fInterceptNM)
3531	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3532	else
3533	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3534
3535	if (fInterceptMF)
3536	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3537	else
3538	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3539
3540	/* Additional intercepts for debugging, define these yourself explicitly. */
3541	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3542	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3543	\| RT_BIT(X86_XCPT_BP)
3544	\| RT_BIT(X86_XCPT_DB)
3545	\| RT_BIT(X86_XCPT_DE)
3546	\| RT_BIT(X86_XCPT_NM)
3547	\| RT_BIT(X86_XCPT_UD)
3548	\| RT_BIT(X86_XCPT_NP)
3549	\| RT_BIT(X86_XCPT_SS)
3550	\| RT_BIT(X86_XCPT_GP)
3551	\| RT_BIT(X86_XCPT_PF)
3552	\| RT_BIT(X86_XCPT_MF)
3553	;
3554	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3555	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3556	#endif
3557
3558	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3559
3560	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3561	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3562	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3563	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3564	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3565	else
3566	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3567
3568	u32GuestCR0 \|= uSetCR0;
3569	u32GuestCR0 &= uZapCR0;
3570	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3571
3572	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3573	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3574	AssertRCReturn(rc, rc);
3575	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3576	AssertRCReturn(rc, rc);
3577	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3578
3579	/*
3580	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3581	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3582	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3583	*/
3584	uint32_t u32CR0Mask = 0;
3585	u32CR0Mask = X86_CR0_PE
3586	\| X86_CR0_NE
3587	\| X86_CR0_WP
3588	\| X86_CR0_PG
3589	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3590	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3591	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3592
3593	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3594	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3595	* and @bugref{6944}. */
3596	#if 0
3597	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3598	u32CR0Mask &= ~X86_CR0_PE;
3599	#endif
3600	if (pVM->hm.s.fNestedPaging)
3601	u32CR0Mask &= ~X86_CR0_WP;
3602
3603	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3604	if (fInterceptNM)
3605	{
3606	u32CR0Mask \|= X86_CR0_TS
3607	\| X86_CR0_MP;
3608	}
3609
3610	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3611	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3612	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3613	AssertRCReturn(rc, rc);
3614	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3615
3616	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3617	}
3618	return rc;
3619	}
3620
3621
3622	/**
3623	* Loads the guest control registers (CR3, CR4) into the guest-state area
3624	* in the VMCS.
3625	*
3626	* @returns VBox status code.
3627	* @param pVM Pointer to the VM.
3628	* @param pVCpu Pointer to the VMCPU.
3629	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3630	* out-of-sync. Make sure to update the required fields
3631	* before using them.
3632	*
3633	* @remarks No-long-jump zone!!!
3634	*/
3635	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3636	{
3637	int rc = VINF_SUCCESS;
3638	PVM pVM = pVCpu->CTX_SUFF(pVM);
3639
3640	/*
3641	* Guest CR2.
3642	* It's always loaded in the assembler code. Nothing to do here.
3643	*/
3644
3645	/*
3646	* Guest CR3.
3647	*/
3648	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3649	{
3650	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3651	if (pVM->hm.s.fNestedPaging)
3652	{
3653	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3654
3655	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3656	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3657	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3658	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3659
3660	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3661	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3662	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3663
3664	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3665	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3666	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3667	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3668
3669	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3670	AssertRCReturn(rc, rc);
3671	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3672
3673	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3674	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3675	{
3676	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3677	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3678	{
3679	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3680	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3681	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3682	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3683	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3684	}
3685
3686	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3687	have Unrestricted Execution to handle the guest when it's not using paging. */
3688	GCPhysGuestCR3 = pMixedCtx->cr3;
3689	}
3690	else
3691	{
3692	/*
3693	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3694	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3695	* EPT takes care of translating it to host-physical addresses.
3696	*/
3697	RTGCPHYS GCPhys;
3698	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3699	Assert(PDMVmmDevHeapIsEnabled(pVM));
3700
3701	/* We obtain it here every time as the guest could have relocated this PCI region. */
3702	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3703	AssertRCReturn(rc, rc);
3704
3705	GCPhysGuestCR3 = GCPhys;
3706	}
3707
3708	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3709	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3710	}
3711	else
3712	{
3713	/* Non-nested paging case, just use the hypervisor's CR3. */
3714	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3715
3716	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3717	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3718	}
3719	AssertRCReturn(rc, rc);
3720
3721	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3722	}
3723
3724	/*
3725	* Guest CR4.
3726	*/
3727	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3728	{
3729	Assert(!(pMixedCtx->cr4 >> 32));
3730	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3731
3732	/* The guest's view of its CR4 is unblemished. */
3733	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3734	AssertRCReturn(rc, rc);
3735	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3736
3737	/* Setup VT-x's view of the guest CR4. */
3738	/*
3739	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3740	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3741	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3742	*/
3743	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3744	{
3745	Assert(pVM->hm.s.vmx.pRealModeTSS);
3746	Assert(PDMVmmDevHeapIsEnabled(pVM));
3747	u32GuestCR4 &= ~X86_CR4_VME;
3748	}
3749
3750	if (pVM->hm.s.fNestedPaging)
3751	{
3752	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3753	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3754	{
3755	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3756	u32GuestCR4 \|= X86_CR4_PSE;
3757	/* Our identity mapping is a 32-bit page directory. */
3758	u32GuestCR4 &= ~X86_CR4_PAE;
3759	}
3760	/* else use guest CR4.*/
3761	}
3762	else
3763	{
3764	/*
3765	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3766	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3767	*/
3768	switch (pVCpu->hm.s.enmShadowMode)
3769	{
3770	case PGMMODE_REAL: /* Real-mode. */
3771	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3772	case PGMMODE_32_BIT: /* 32-bit paging. */
3773	{
3774	u32GuestCR4 &= ~X86_CR4_PAE;
3775	break;
3776	}
3777
3778	case PGMMODE_PAE: /* PAE paging. */
3779	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3780	{
3781	u32GuestCR4 \|= X86_CR4_PAE;
3782	break;
3783	}
3784
3785	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3786	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3787	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3788	break;
3789	#endif
3790	default:
3791	AssertFailed();
3792	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3793	}
3794	}
3795
3796	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3797	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3798	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3799	u32GuestCR4 \|= uSetCR4;
3800	u32GuestCR4 &= uZapCR4;
3801
3802	/* Write VT-x's view of the guest CR4 into the VMCS. */
3803	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3804	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3805	AssertRCReturn(rc, rc);
3806
3807	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3808	uint32_t u32CR4Mask = 0;
3809	u32CR4Mask = X86_CR4_VME
3810	\| X86_CR4_PAE
3811	\| X86_CR4_PGE
3812	\| X86_CR4_PSE
3813	\| X86_CR4_VMXE;
3814	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3815	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3816	AssertRCReturn(rc, rc);
3817
3818	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3819	}
3820	return rc;
3821	}
3822
3823
3824	/**
3825	* Loads the guest debug registers into the guest-state area in the VMCS.
3826	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3827	*
3828	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3829	*
3830	* @returns VBox status code.
3831	* @param pVCpu Pointer to the VMCPU.
3832	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3833	* out-of-sync. Make sure to update the required fields
3834	* before using them.
3835	*
3836	* @remarks No-long-jump zone!!!
3837	*/
3838	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3839	{
3840	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3841	return VINF_SUCCESS;
3842
3843	#ifdef VBOX_STRICT
3844	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3845	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3846	{
3847	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3848	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3849	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3850	}
3851	#endif
3852
3853	int rc;
3854	PVM pVM = pVCpu->CTX_SUFF(pVM);
3855	bool fInterceptDB = false;
3856	bool fInterceptMovDRx = false;
3857	if ( pVCpu->hm.s.fSingleInstruction
3858	\|\| DBGFIsStepping(pVCpu))
3859	{
3860	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3861	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3862	{
3863	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3864	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3865	AssertRCReturn(rc, rc);
3866	Assert(fInterceptDB == false);
3867	}
3868	else
3869	{
3870	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3871	pVCpu->hm.s.fClearTrapFlag = true;
3872	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3873	fInterceptDB = true;
3874	}
3875	}
3876
3877	if ( fInterceptDB
3878	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3879	{
3880	/*
3881	* Use the combined guest and host DRx values found in the hypervisor
3882	* register set because the debugger has breakpoints active or someone
3883	* is single stepping on the host side without a monitor trap flag.
3884	*
3885	* Note! DBGF expects a clean DR6 state before executing guest code.
3886	*/
3887	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3888	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3889	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3890	{
3891	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3892	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3893	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3894	}
3895	else
3896	#endif
3897	if (!CPUMIsHyperDebugStateActive(pVCpu))
3898	{
3899	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3900	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3901	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3902	}
3903
3904	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3905	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3906	AssertRCReturn(rc, rc);
3907
3908	pVCpu->hm.s.fUsingHyperDR7 = true;
3909	fInterceptDB = true;
3910	fInterceptMovDRx = true;
3911	}
3912	else
3913	{
3914	/*
3915	* If the guest has enabled debug registers, we need to load them prior to
3916	* executing guest code so they'll trigger at the right time.
3917	*/
3918	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3919	{
3920	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3921	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3922	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3923	{
3924	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3925	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3926	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3927	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3928	}
3929	else
3930	#endif
3931	if (!CPUMIsGuestDebugStateActive(pVCpu))
3932	{
3933	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3934	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3935	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3936	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3937	}
3938	Assert(!fInterceptDB);
3939	Assert(!fInterceptMovDRx);
3940	}
3941	/*
3942	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3943	* must intercept #DB in order to maintain a correct DR6 guest value.
3944	*/
3945	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3946	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3947	&& !CPUMIsGuestDebugStateActive(pVCpu))
3948	#else
3949	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3950	#endif
3951	{
3952	fInterceptMovDRx = true;
3953	fInterceptDB = true;
3954	}
3955
3956	/* Update guest DR7. */
3957	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3958	AssertRCReturn(rc, rc);
3959
3960	pVCpu->hm.s.fUsingHyperDR7 = false;
3961	}
3962
3963	/*
3964	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3965	*/
3966	if (fInterceptDB)
3967	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3968	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3969	{
3970	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3971	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3972	#endif
3973	}
3974	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3975	AssertRCReturn(rc, rc);
3976
3977	/*
3978	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3979	*/
3980	if (fInterceptMovDRx)
3981	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3982	else
3983	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3984	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3985	AssertRCReturn(rc, rc);
3986
3987	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3988	return VINF_SUCCESS;
3989	}
3990
3991
3992	#ifdef VBOX_STRICT
3993	/**
3994	* Strict function to validate segment registers.
3995	*
3996	* @remarks ASSUMES CR0 is up to date.
3997	*/
3998	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3999	{
4000	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4001	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4002	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4003	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4004	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4005	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4006	{
4007	/* Protected mode checks */
4008	/* CS */
4009	Assert(pCtx->cs.Attr.n.u1Present);
4010	Assert(!(pCtx->cs.Attr.u & 0xf00));
4011	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4012	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4013	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4014	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4015	\|\| (pCtx->cs.Attr.n.u1Granularity));
4016	/* CS cannot be loaded with NULL in protected mode. */
4017	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4018	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4019	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4020	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4021	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4022	else
4023	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4024	/* SS */
4025	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4026	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4027	if ( !(pCtx->cr0 & X86_CR0_PE)
4028	\|\| pCtx->cs.Attr.n.u4Type == 3)
4029	{
4030	Assert(!pCtx->ss.Attr.n.u2Dpl);
4031	}
4032	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4033	{
4034	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4035	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4036	Assert(pCtx->ss.Attr.n.u1Present);
4037	Assert(!(pCtx->ss.Attr.u & 0xf00));
4038	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4039	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4040	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4041	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4042	\|\| (pCtx->ss.Attr.n.u1Granularity));
4043	}
4044	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4045	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4046	{
4047	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4048	Assert(pCtx->ds.Attr.n.u1Present);
4049	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4050	Assert(!(pCtx->ds.Attr.u & 0xf00));
4051	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4052	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4053	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4054	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4055	\|\| (pCtx->ds.Attr.n.u1Granularity));
4056	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4057	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4058	}
4059	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4060	{
4061	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4062	Assert(pCtx->es.Attr.n.u1Present);
4063	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4064	Assert(!(pCtx->es.Attr.u & 0xf00));
4065	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4066	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4067	\|\| !(pCtx->es.Attr.n.u1Granularity));
4068	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4069	\|\| (pCtx->es.Attr.n.u1Granularity));
4070	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4071	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4072	}
4073	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4074	{
4075	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4076	Assert(pCtx->fs.Attr.n.u1Present);
4077	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4078	Assert(!(pCtx->fs.Attr.u & 0xf00));
4079	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4080	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4081	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4082	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4083	\|\| (pCtx->fs.Attr.n.u1Granularity));
4084	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4085	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4086	}
4087	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4088	{
4089	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4090	Assert(pCtx->gs.Attr.n.u1Present);
4091	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4092	Assert(!(pCtx->gs.Attr.u & 0xf00));
4093	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4094	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4095	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4096	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4097	\|\| (pCtx->gs.Attr.n.u1Granularity));
4098	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4099	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4100	}
4101	/* 64-bit capable CPUs. */
4102	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4103	Assert(!(pCtx->cs.u64Base >> 32));
4104	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4105	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4106	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4107	# endif
4108	}
4109	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4110	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4111	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4112	{
4113	/* Real and v86 mode checks. */
4114	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4115	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4116	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4117	{
4118	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4119	}
4120	else
4121	{
4122	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4123	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4124	}
4125
4126	/* CS */
4127	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4128	Assert(pCtx->cs.u32Limit == 0xffff);
4129	Assert(u32CSAttr == 0xf3);
4130	/* SS */
4131	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4132	Assert(pCtx->ss.u32Limit == 0xffff);
4133	Assert(u32SSAttr == 0xf3);
4134	/* DS */
4135	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4136	Assert(pCtx->ds.u32Limit == 0xffff);
4137	Assert(u32DSAttr == 0xf3);
4138	/* ES */
4139	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4140	Assert(pCtx->es.u32Limit == 0xffff);
4141	Assert(u32ESAttr == 0xf3);
4142	/* FS */
4143	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4144	Assert(pCtx->fs.u32Limit == 0xffff);
4145	Assert(u32FSAttr == 0xf3);
4146	/* GS */
4147	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4148	Assert(pCtx->gs.u32Limit == 0xffff);
4149	Assert(u32GSAttr == 0xf3);
4150	/* 64-bit capable CPUs. */
4151	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4152	Assert(!(pCtx->cs.u64Base >> 32));
4153	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4154	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4155	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4156	# endif
4157	}
4158	}
4159	#endif /* VBOX_STRICT */
4160
4161
4162	/**
4163	* Writes a guest segment register into the guest-state area in the VMCS.
4164	*
4165	* @returns VBox status code.
4166	* @param pVCpu Pointer to the VMCPU.
4167	* @param idxSel Index of the selector in the VMCS.
4168	* @param idxLimit Index of the segment limit in the VMCS.
4169	* @param idxBase Index of the segment base in the VMCS.
4170	* @param idxAccess Index of the access rights of the segment in the VMCS.
4171	* @param pSelReg Pointer to the segment selector.
4172	*
4173	* @remarks No-long-jump zone!!!
4174	*/
4175	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4176	uint32_t idxAccess, PCPUMSELREG pSelReg)
4177	{
4178	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4179	AssertRCReturn(rc, rc);
4180	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4181	AssertRCReturn(rc, rc);
4182	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4183	AssertRCReturn(rc, rc);
4184
4185	uint32_t u32Access = pSelReg->Attr.u;
4186	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4187	{
4188	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4189	u32Access = 0xf3;
4190	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4191	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4192	}
4193	else
4194	{
4195	/*
4196	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4197	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4198	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4199	* loaded in protected-mode have their attribute as 0.
4200	*/
4201	if (!u32Access)
4202	u32Access = X86DESCATTR_UNUSABLE;
4203	}
4204
4205	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4206	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4207	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4208
4209	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4210	AssertRCReturn(rc, rc);
4211	return rc;
4212	}
4213
4214
4215	/**
4216	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4217	* into the guest-state area in the VMCS.
4218	*
4219	* @returns VBox status code.
4220	* @param pVM Pointer to the VM.
4221	* @param pVCPU Pointer to the VMCPU.
4222	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4223	* out-of-sync. Make sure to update the required fields
4224	* before using them.
4225	*
4226	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4227	* @remarks No-long-jump zone!!!
4228	*/
4229	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4230	{
4231	int rc = VERR_INTERNAL_ERROR_5;
4232	PVM pVM = pVCpu->CTX_SUFF(pVM);
4233
4234	/*
4235	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4236	*/
4237	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4238	{
4239	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4240	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4241	{
4242	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4243	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4244	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4245	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4246	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4247	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4248	}
4249
4250	#ifdef VBOX_WITH_REM
4251	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4252	{
4253	Assert(pVM->hm.s.vmx.pRealModeTSS);
4254	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4255	if ( pVCpu->hm.s.vmx.fWasInRealMode
4256	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4257	{
4258	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4259	in real-mode (e.g. OpenBSD 4.0) */
4260	REMFlushTBs(pVM);
4261	Log4(("Load: Switch to protected mode detected!\n"));
4262	pVCpu->hm.s.vmx.fWasInRealMode = false;
4263	}
4264	}
4265	#endif
4266	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4267	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4268	AssertRCReturn(rc, rc);
4269	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4270	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4271	AssertRCReturn(rc, rc);
4272	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4273	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4274	AssertRCReturn(rc, rc);
4275	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4276	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4277	AssertRCReturn(rc, rc);
4278	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4279	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4280	AssertRCReturn(rc, rc);
4281	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4282	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4283	AssertRCReturn(rc, rc);
4284
4285	#ifdef VBOX_STRICT
4286	/* Validate. */
4287	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4288	#endif
4289
4290	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4291	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4292	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4293	}
4294
4295	/*
4296	* Guest TR.
4297	*/
4298	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4299	{
4300	/*
4301	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4302	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4303	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4304	*/
4305	uint16_t u16Sel = 0;
4306	uint32_t u32Limit = 0;
4307	uint64_t u64Base = 0;
4308	uint32_t u32AccessRights = 0;
4309
4310	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4311	{
4312	u16Sel = pMixedCtx->tr.Sel;
4313	u32Limit = pMixedCtx->tr.u32Limit;
4314	u64Base = pMixedCtx->tr.u64Base;
4315	u32AccessRights = pMixedCtx->tr.Attr.u;
4316	}
4317	else
4318	{
4319	Assert(pVM->hm.s.vmx.pRealModeTSS);
4320	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4321
4322	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4323	RTGCPHYS GCPhys;
4324	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4325	AssertRCReturn(rc, rc);
4326
4327	X86DESCATTR DescAttr;
4328	DescAttr.u = 0;
4329	DescAttr.n.u1Present = 1;
4330	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4331
4332	u16Sel = 0;
4333	u32Limit = HM_VTX_TSS_SIZE;
4334	u64Base = GCPhys; /* in real-mode phys = virt. */
4335	u32AccessRights = DescAttr.u;
4336	}
4337
4338	/* Validate. */
4339	Assert(!(u16Sel & RT_BIT(2)));
4340	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4341	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4342	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4343	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4344	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4345	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4346	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4347	Assert( (u32Limit & 0xfff) == 0xfff
4348	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4349	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4350	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4351
4352	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4353	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4354	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4355	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4356
4357	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4358	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4359	}
4360
4361	/*
4362	* Guest GDTR.
4363	*/
4364	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4365	{
4366	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4367	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4368
4369	/* Validate. */
4370	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4371
4372	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4373	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4374	}
4375
4376	/*
4377	* Guest LDTR.
4378	*/
4379	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4380	{
4381	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4382	uint32_t u32Access = 0;
4383	if (!pMixedCtx->ldtr.Attr.u)
4384	u32Access = X86DESCATTR_UNUSABLE;
4385	else
4386	u32Access = pMixedCtx->ldtr.Attr.u;
4387
4388	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4389	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4390	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4391	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4392
4393	/* Validate. */
4394	if (!(u32Access & X86DESCATTR_UNUSABLE))
4395	{
4396	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4397	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4398	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4399	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4400	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4401	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4402	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4403	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4404	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4405	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4406	}
4407
4408	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4409	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4410	}
4411
4412	/*
4413	* Guest IDTR.
4414	*/
4415	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4416	{
4417	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4418	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4419
4420	/* Validate. */
4421	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4422
4423	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4424	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4425	}
4426
4427	return VINF_SUCCESS;
4428	}
4429
4430
4431	/**
4432	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4433	* areas. These MSRs will automatically be loaded to the host CPU on every
4434	* successful VM entry and stored from the host CPU on every successful VM exit.
4435	*
4436	* This also creates/updates MSR slots for the host MSRs. The actual host
4437	* MSR values are -not- updated here for performance reasons. See
4438	* hmR0VmxSaveHostMsrs().
4439	*
4440	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4441	*
4442	* @returns VBox status code.
4443	* @param pVCpu Pointer to the VMCPU.
4444	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4445	* out-of-sync. Make sure to update the required fields
4446	* before using them.
4447	*
4448	* @remarks No-long-jump zone!!!
4449	*/
4450	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4451	{
4452	AssertPtr(pVCpu);
4453	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4454
4455	/*
4456	* MSRs that we use the auto-load/store MSR area in the VMCS.
4457	*/
4458	PVM pVM = pVCpu->CTX_SUFF(pVM);
4459	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4460	{
4461	if (pVM->hm.s.fAllow64BitGuests)
4462	{
4463	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4464	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4465	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4466	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4467	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4468	# ifdef DEBUG
4469	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4470	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4471	Log4(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4472	# endif
4473	#endif
4474	}
4475	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4476	}
4477
4478	/*
4479	* Guest Sysenter MSRs.
4480	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4481	* VM-exits on WRMSRs for these MSRs.
4482	*/
4483	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4484	{
4485	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4486	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4487	}
4488
4489	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4490	{
4491	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4492	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4493	}
4494
4495	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4496	{
4497	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4498	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4499	}
4500
4501	return VINF_SUCCESS;
4502	}
4503
4504
4505	/**
4506	* Loads the guest activity state into the guest-state area in the VMCS.
4507	*
4508	* @returns VBox status code.
4509	* @param pVCpu Pointer to the VMCPU.
4510	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4511	* out-of-sync. Make sure to update the required fields
4512	* before using them.
4513	*
4514	* @remarks No-long-jump zone!!!
4515	*/
4516	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4517	{
4518	NOREF(pCtx);
4519	/** @todo See if we can make use of other states, e.g.
4520	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4521	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4522	{
4523	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4524	AssertRCReturn(rc, rc);
4525
4526	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4527	}
4528	return VINF_SUCCESS;
4529	}
4530
4531
4532	/**
4533	* Sets up the appropriate function to run guest code.
4534	*
4535	* @returns VBox status code.
4536	* @param pVCpu Pointer to the VMCPU.
4537	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4538	* out-of-sync. Make sure to update the required fields
4539	* before using them.
4540	*
4541	* @remarks No-long-jump zone!!!
4542	*/
4543	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4544	{
4545	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4546	{
4547	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4548	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4549	#endif
4550	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4551	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4552	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4553	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4554	{
4555	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4556	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4557	}
4558	#else
4559	/* 64-bit host or hybrid host. */
4560	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4561	#endif
4562	}
4563	else
4564	{
4565	/* Guest is not in long mode, use the 32-bit handler. */
4566	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4567	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4568	{
4569	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4570	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4571	}
4572	#else
4573	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4574	#endif
4575	}
4576	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4577	return VINF_SUCCESS;
4578	}
4579
4580
4581	/**
4582	* Wrapper for running the guest code in VT-x.
4583	*
4584	* @returns VBox strict status code.
4585	* @param pVM Pointer to the VM.
4586	* @param pVCpu Pointer to the VMCPU.
4587	* @param pCtx Pointer to the guest-CPU context.
4588	*
4589	* @remarks No-long-jump zone!!!
4590	*/
4591	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4592	{
4593	/*
4594	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4595	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4596	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4597	*/
4598	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4599	/** @todo Add stats for resume vs launch. */
4600	#ifdef VBOX_WITH_KERNEL_USING_XMM
4601	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4602	#else
4603	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4604	#endif
4605	}
4606
4607
4608	/**
4609	* Reports world-switch error and dumps some useful debug info.
4610	*
4611	* @param pVM Pointer to the VM.
4612	* @param pVCpu Pointer to the VMCPU.
4613	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4614	* @param pCtx Pointer to the guest-CPU context.
4615	* @param pVmxTransient Pointer to the VMX transient structure (only
4616	* exitReason updated).
4617	*/
4618	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4619	{
4620	Assert(pVM);
4621	Assert(pVCpu);
4622	Assert(pCtx);
4623	Assert(pVmxTransient);
4624	HMVMX_ASSERT_PREEMPT_SAFE();
4625
4626	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4627	switch (rcVMRun)
4628	{
4629	case VERR_VMX_INVALID_VMXON_PTR:
4630	AssertFailed();
4631	break;
4632	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4633	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4634	{
4635	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4636	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4637	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4638	AssertRC(rc);
4639
4640	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4641	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4642	Cannot do it here as we may have been long preempted. */
4643
4644	#ifdef VBOX_STRICT
4645	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4646	pVmxTransient->uExitReason));
4647	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4648	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4649	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4650	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4651	else
4652	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4653	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4654	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4655
4656	/* VMX control bits. */
4657	uint32_t u32Val;
4658	uint64_t u64Val;
4659	HMVMXHCUINTREG uHCReg;
4660	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4661	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4662	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4663	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4664	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4665	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4666	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4667	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4668	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4669	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4670	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4671	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4672	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4673	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4674	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4675	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4676	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4677	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4678	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4679	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4680	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4681	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4682	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4683	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4684	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4685	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4686	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4687	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4688	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4689	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4690	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4691	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4692	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4693	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4694	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4695	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4696	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4697	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4698	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4699	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4700	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4701	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4702
4703	/* Guest bits. */
4704	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4705	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4706	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4707	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4708	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4709	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4710	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4711	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4712
4713	/* Host bits. */
4714	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4715	Log4(("Host CR0 %#RHr\n", uHCReg));
4716	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4717	Log4(("Host CR3 %#RHr\n", uHCReg));
4718	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4719	Log4(("Host CR4 %#RHr\n", uHCReg));
4720
4721	RTGDTR HostGdtr;
4722	PCX86DESCHC pDesc;
4723	ASMGetGDTR(&HostGdtr);
4724	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4725	Log4(("Host CS %#08x\n", u32Val));
4726	if (u32Val < HostGdtr.cbGdt)
4727	{
4728	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4729	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4730	}
4731
4732	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4733	Log4(("Host DS %#08x\n", u32Val));
4734	if (u32Val < HostGdtr.cbGdt)
4735	{
4736	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4737	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4738	}
4739
4740	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4741	Log4(("Host ES %#08x\n", u32Val));
4742	if (u32Val < HostGdtr.cbGdt)
4743	{
4744	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4745	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4746	}
4747
4748	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4749	Log4(("Host FS %#08x\n", u32Val));
4750	if (u32Val < HostGdtr.cbGdt)
4751	{
4752	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4753	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4754	}
4755
4756	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4757	Log4(("Host GS %#08x\n", u32Val));
4758	if (u32Val < HostGdtr.cbGdt)
4759	{
4760	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4761	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4762	}
4763
4764	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4765	Log4(("Host SS %#08x\n", u32Val));
4766	if (u32Val < HostGdtr.cbGdt)
4767	{
4768	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4769	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4770	}
4771
4772	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4773	Log4(("Host TR %#08x\n", u32Val));
4774	if (u32Val < HostGdtr.cbGdt)
4775	{
4776	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4777	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4778	}
4779
4780	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4781	Log4(("Host TR Base %#RHv\n", uHCReg));
4782	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4783	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4784	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4785	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4786	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4787	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4788	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4789	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4790	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4791	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4792	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4793	Log4(("Host RSP %#RHv\n", uHCReg));
4794	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4795	Log4(("Host RIP %#RHv\n", uHCReg));
4796	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4797	if (HMVMX_IS_64BIT_HOST_MODE())
4798	{
4799	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4800	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4801	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4802	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4803	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4804	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4805	}
4806	# endif
4807	#endif /* VBOX_STRICT */
4808	break;
4809	}
4810
4811	default:
4812	/* Impossible */
4813	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4814	break;
4815	}
4816	NOREF(pVM); NOREF(pCtx);
4817	}
4818
4819
4820	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4821	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4822	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4823	#endif
4824	#ifdef VBOX_STRICT
4825	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4826	{
4827	switch (idxField)
4828	{
4829	case VMX_VMCS_GUEST_RIP:
4830	case VMX_VMCS_GUEST_RSP:
4831	case VMX_VMCS_GUEST_SYSENTER_EIP:
4832	case VMX_VMCS_GUEST_SYSENTER_ESP:
4833	case VMX_VMCS_GUEST_GDTR_BASE:
4834	case VMX_VMCS_GUEST_IDTR_BASE:
4835	case VMX_VMCS_GUEST_CS_BASE:
4836	case VMX_VMCS_GUEST_DS_BASE:
4837	case VMX_VMCS_GUEST_ES_BASE:
4838	case VMX_VMCS_GUEST_FS_BASE:
4839	case VMX_VMCS_GUEST_GS_BASE:
4840	case VMX_VMCS_GUEST_SS_BASE:
4841	case VMX_VMCS_GUEST_LDTR_BASE:
4842	case VMX_VMCS_GUEST_TR_BASE:
4843	case VMX_VMCS_GUEST_CR3:
4844	return true;
4845	}
4846	return false;
4847	}
4848
4849	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4850	{
4851	switch (idxField)
4852	{
4853	/* Read-only fields. */
4854	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4855	return true;
4856	}
4857	/* Remaining readable fields should also be writable. */
4858	return hmR0VmxIsValidWriteField(idxField);
4859	}
4860	#endif /* VBOX_STRICT */
4861
4862
4863	/**
4864	* Executes the specified handler in 64-bit mode.
4865	*
4866	* @returns VBox status code.
4867	* @param pVM Pointer to the VM.
4868	* @param pVCpu Pointer to the VMCPU.
4869	* @param pCtx Pointer to the guest CPU context.
4870	* @param enmOp The operation to perform.
4871	* @param cbParam Number of parameters.
4872	* @param paParam Array of 32-bit parameters.
4873	*/
4874	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4875	uint32_t *paParam)
4876	{
4877	int rc, rc2;
4878	PHMGLOBALCPUINFO pCpu;
4879	RTHCPHYS HCPhysCpuPage;
4880	RTCCUINTREG uOldEflags;
4881
4882	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4883	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4884	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4885	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4886
4887	#ifdef VBOX_STRICT
4888	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4889	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4890
4891	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4892	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4893	#endif
4894
4895	/* Disable interrupts. */
4896	uOldEflags = ASMIntDisableFlags();
4897
4898	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4899	RTCPUID idHostCpu = RTMpCpuId();
4900	CPUMR0SetLApic(pVCpu, idHostCpu);
4901	#endif
4902
4903	pCpu = HMR0GetCurrentCpu();
4904	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4905
4906	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4907	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4908
4909	/* Leave VMX Root Mode. */
4910	VMXDisable();
4911
4912	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4913
4914	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4915	CPUMSetHyperEIP(pVCpu, enmOp);
4916	for (int i = (int)cbParam - 1; i >= 0; i--)
4917	CPUMPushHyper(pVCpu, paParam[i]);
4918
4919	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4920
4921	/* Call the switcher. */
4922	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4923	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4924
4925	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4926	/* Make sure the VMX instructions don't cause #UD faults. */
4927	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4928
4929	/* Re-enter VMX Root Mode */
4930	rc2 = VMXEnable(HCPhysCpuPage);
4931	if (RT_FAILURE(rc2))
4932	{
4933	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4934	ASMSetFlags(uOldEflags);
4935	return rc2;
4936	}
4937
4938	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4939	AssertRC(rc2);
4940	Assert(!(ASMGetFlags() & X86_EFL_IF));
4941	ASMSetFlags(uOldEflags);
4942	return rc;
4943	}
4944
4945
4946	/**
4947	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4948	* supporting 64-bit guests.
4949	*
4950	* @returns VBox status code.
4951	* @param fResume Whether to VMLAUNCH or VMRESUME.
4952	* @param pCtx Pointer to the guest-CPU context.
4953	* @param pCache Pointer to the VMCS cache.
4954	* @param pVM Pointer to the VM.
4955	* @param pVCpu Pointer to the VMCPU.
4956	*/
4957	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4958	{
4959	uint32_t aParam[6];
4960	PHMGLOBALCPUINFO pCpu = NULL;
4961	RTHCPHYS HCPhysCpuPage = 0;
4962	int rc = VERR_INTERNAL_ERROR_5;
4963
4964	pCpu = HMR0GetCurrentCpu();
4965	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4966
4967	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4968	pCache->uPos = 1;
4969	pCache->interPD = PGMGetInterPaeCR3(pVM);
4970	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4971	#endif
4972
4973	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4974	pCache->TestIn.HCPhysCpuPage = 0;
4975	pCache->TestIn.HCPhysVmcs = 0;
4976	pCache->TestIn.pCache = 0;
4977	pCache->TestOut.HCPhysVmcs = 0;
4978	pCache->TestOut.pCache = 0;
4979	pCache->TestOut.pCtx = 0;
4980	pCache->TestOut.eflags = 0;
4981	#endif
4982
4983	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4984	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4985	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4986	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4987	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4988	aParam[5] = 0;
4989
4990	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4991	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4992	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4993	#endif
4994	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4995
4996	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4997	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4998	Assert(pCtx->dr[4] == 10);
4999	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5000	#endif
5001
5002	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5003	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5004	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5005	pVCpu->hm.s.vmx.HCPhysVmcs));
5006	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5007	pCache->TestOut.HCPhysVmcs));
5008	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5009	pCache->TestOut.pCache));
5010	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5011	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5012	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5013	pCache->TestOut.pCtx));
5014	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5015	#endif
5016	return rc;
5017	}
5018
5019
5020	/**
5021	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5022	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5023	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5024	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5025	*
5026	* @returns VBox status code.
5027	* @param pVM Pointer to the VM.
5028	* @param pVCpu Pointer to the VMCPU.
5029	*/
5030	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5031	{
5032	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5033	{ \
5034	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5035	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5036	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5037	++cReadFields; \
5038	}
5039
5040	AssertPtr(pVM);
5041	AssertPtr(pVCpu);
5042	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5043	uint32_t cReadFields = 0;
5044
5045	/*
5046	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5047	* and serve to indicate exceptions to the rules.
5048	*/
5049
5050	/* Guest-natural selector base fields. */
5051	#if 0
5052	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5053	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5054	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5055	#endif
5056	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5057	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5058	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5059	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5060	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5061	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5062	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5063	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5064	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5065	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5066	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5067	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5068	#if 0
5069	/* Unused natural width guest-state fields. */
5070	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5071	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5072	#endif
5073	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5074	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5075
5076	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5077	#if 0
5078	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5079	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5080	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5081	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5082	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5083	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5084	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5085	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5086	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5087	#endif
5088
5089	/* Natural width guest-state fields. */
5090	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5091	#if 0
5092	/* Currently unused field. */
5093	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5094	#endif
5095
5096	if (pVM->hm.s.fNestedPaging)
5097	{
5098	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5099	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5100	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5101	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5102	}
5103	else
5104	{
5105	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5106	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5107	}
5108
5109	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5110	return VINF_SUCCESS;
5111	}
5112
5113
5114	/**
5115	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5116	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5117	* darwin, running 64-bit guests).
5118	*
5119	* @returns VBox status code.
5120	* @param pVCpu Pointer to the VMCPU.
5121	* @param idxField The VMCS field encoding.
5122	* @param u64Val 16, 32 or 64-bit value.
5123	*/
5124	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5125	{
5126	int rc;
5127	switch (idxField)
5128	{
5129	/*
5130	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5131	*/
5132	/* 64-bit Control fields. */
5133	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5134	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5135	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5136	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5137	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5138	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5139	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5140	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5141	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5142	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5143	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5144	case VMX_VMCS64_CTRL_EPTP_FULL:
5145	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5146	/* 64-bit Guest-state fields. */
5147	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5148	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5149	case VMX_VMCS64_GUEST_PAT_FULL:
5150	case VMX_VMCS64_GUEST_EFER_FULL:
5151	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5152	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5153	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5154	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5155	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5156	/* 64-bit Host-state fields. */
5157	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5158	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5159	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5160	{
5161	rc = VMXWriteVmcs32(idxField, u64Val);
5162	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5163	break;
5164	}
5165
5166	/*
5167	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5168	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5169	*/
5170	/* Natural-width Guest-state fields. */
5171	case VMX_VMCS_GUEST_CR3:
5172	case VMX_VMCS_GUEST_ES_BASE:
5173	case VMX_VMCS_GUEST_CS_BASE:
5174	case VMX_VMCS_GUEST_SS_BASE:
5175	case VMX_VMCS_GUEST_DS_BASE:
5176	case VMX_VMCS_GUEST_FS_BASE:
5177	case VMX_VMCS_GUEST_GS_BASE:
5178	case VMX_VMCS_GUEST_LDTR_BASE:
5179	case VMX_VMCS_GUEST_TR_BASE:
5180	case VMX_VMCS_GUEST_GDTR_BASE:
5181	case VMX_VMCS_GUEST_IDTR_BASE:
5182	case VMX_VMCS_GUEST_RSP:
5183	case VMX_VMCS_GUEST_RIP:
5184	case VMX_VMCS_GUEST_SYSENTER_ESP:
5185	case VMX_VMCS_GUEST_SYSENTER_EIP:
5186	{
5187	if (!(u64Val >> 32))
5188	{
5189	/* If this field is 64-bit, VT-x will zero out the top bits. */
5190	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5191	}
5192	else
5193	{
5194	/* Assert that only the 32->64 switcher case should ever come here. */
5195	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5196	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5197	}
5198	break;
5199	}
5200
5201	default:
5202	{
5203	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5204	rc = VERR_INVALID_PARAMETER;
5205	break;
5206	}
5207	}
5208	AssertRCReturn(rc, rc);
5209	return rc;
5210	}
5211
5212
5213	/**
5214	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5215	* hosts (except darwin) for 64-bit guests.
5216	*
5217	* @param pVCpu Pointer to the VMCPU.
5218	* @param idxField The VMCS field encoding.
5219	* @param u64Val 16, 32 or 64-bit value.
5220	*/
5221	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5222	{
5223	AssertPtr(pVCpu);
5224	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5225
5226	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5227	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5228
5229	/* Make sure there are no duplicates. */
5230	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5231	{
5232	if (pCache->Write.aField[i] == idxField)
5233	{
5234	pCache->Write.aFieldVal[i] = u64Val;
5235	return VINF_SUCCESS;
5236	}
5237	}
5238
5239	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5240	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5241	pCache->Write.cValidEntries++;
5242	return VINF_SUCCESS;
5243	}
5244
5245	/* Enable later when the assembly code uses these as callbacks. */
5246	#if 0
5247	/*
5248	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5249	*
5250	* @param pVCpu Pointer to the VMCPU.
5251	* @param pCache Pointer to the VMCS cache.
5252	*
5253	* @remarks No-long-jump zone!!!
5254	*/
5255	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5256	{
5257	AssertPtr(pCache);
5258	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5259	{
5260	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5261	AssertRC(rc);
5262	}
5263	pCache->Write.cValidEntries = 0;
5264	}
5265
5266
5267	/**
5268	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5269	*
5270	* @param pVCpu Pointer to the VMCPU.
5271	* @param pCache Pointer to the VMCS cache.
5272	*
5273	* @remarks No-long-jump zone!!!
5274	*/
5275	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5276	{
5277	AssertPtr(pCache);
5278	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5279	{
5280	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5281	AssertRC(rc);
5282	}
5283	}
5284	#endif
5285	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5286
5287
5288	/**
5289	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5290	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5291	* timer.
5292	*
5293	* @returns VBox status code.
5294	* @param pVCpu Pointer to the VMCPU.
5295	*
5296	* @remarks No-long-jump zone!!!
5297	*/
5298	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5299	{
5300	int rc = VERR_INTERNAL_ERROR_5;
5301	bool fOffsettedTsc = false;
5302	PVM pVM = pVCpu->CTX_SUFF(pVM);
5303	if (pVM->hm.s.vmx.fUsePreemptTimer)
5304	{
5305	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5306
5307	/* Make sure the returned values have sane upper and lower boundaries. */
5308	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5309	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5310	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5311	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5312
5313	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5314	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5315	}
5316	else
5317	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5318
5319	if (fOffsettedTsc)
5320	{
5321	uint64_t u64CurTSC = ASMReadTSC();
5322	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5323	{
5324	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5325	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5326
5327	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5328	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5329	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5330	}
5331	else
5332	{
5333	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5334	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5335	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5336	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5337	}
5338	}
5339	else
5340	{
5341	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5342	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5343	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5344	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5345	}
5346	}
5347
5348
5349	/**
5350	* Determines if an exception is a contributory exception. Contributory
5351	* exceptions are ones which can cause double-faults. Page-fault is
5352	* intentionally not included here as it's a conditional contributory exception.
5353	*
5354	* @returns true if the exception is contributory, false otherwise.
5355	* @param uVector The exception vector.
5356	*/
5357	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5358	{
5359	switch (uVector)
5360	{
5361	case X86_XCPT_GP:
5362	case X86_XCPT_SS:
5363	case X86_XCPT_NP:
5364	case X86_XCPT_TS:
5365	case X86_XCPT_DE:
5366	return true;
5367	default:
5368	break;
5369	}
5370	return false;
5371	}
5372
5373
5374	/**
5375	* Sets an event as a pending event to be injected into the guest.
5376	*
5377	* @param pVCpu Pointer to the VMCPU.
5378	* @param u32IntInfo The VM-entry interruption-information field.
5379	* @param cbInstr The VM-entry instruction length in bytes (for software
5380	* interrupts, exceptions and privileged software
5381	* exceptions).
5382	* @param u32ErrCode The VM-entry exception error code.
5383	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5384	* page-fault.
5385	*
5386	* @remarks Statistics counter assumes this is a guest event being injected or
5387	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5388	* always incremented.
5389	*/
5390	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5391	RTGCUINTPTR GCPtrFaultAddress)
5392	{
5393	Assert(!pVCpu->hm.s.Event.fPending);
5394	pVCpu->hm.s.Event.fPending = true;
5395	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5396	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5397	pVCpu->hm.s.Event.cbInstr = cbInstr;
5398	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5399
5400	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5401	}
5402
5403
5404	/**
5405	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5406	*
5407	* @param pVCpu Pointer to the VMCPU.
5408	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5409	* out-of-sync. Make sure to update the required fields
5410	* before using them.
5411	*/
5412	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5413	{
5414	NOREF(pMixedCtx);
5415	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5416	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5417	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5418	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5419	}
5420
5421
5422	/**
5423	* Handle a condition that occurred while delivering an event through the guest
5424	* IDT.
5425	*
5426	* @returns VBox status code (informational error codes included).
5427	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5428	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5429	* continue execution of the guest which will delivery the #DF.
5430	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5431	*
5432	* @param pVCpu Pointer to the VMCPU.
5433	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5434	* out-of-sync. Make sure to update the required fields
5435	* before using them.
5436	* @param pVmxTransient Pointer to the VMX transient structure.
5437	*
5438	* @remarks No-long-jump zone!!!
5439	*/
5440	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5441	{
5442	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5443	AssertRCReturn(rc, rc);
5444	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5445	{
5446	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5447	AssertRCReturn(rc, rc);
5448
5449	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5450	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5451	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5452
5453	typedef enum
5454	{
5455	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5456	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5457	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5458	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5459	} VMXREFLECTXCPT;
5460
5461	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5462	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5463	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5464	{
5465	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5466	{
5467	enmReflect = VMXREFLECTXCPT_XCPT;
5468	#ifdef VBOX_STRICT
5469	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5470	&& uExitVector == X86_XCPT_PF)
5471	{
5472	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5473	}
5474	#endif
5475	if ( uExitVector == X86_XCPT_PF
5476	&& uIdtVector == X86_XCPT_PF)
5477	{
5478	pVmxTransient->fVectoringPF = true;
5479	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5480	}
5481	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5482	&& hmR0VmxIsContributoryXcpt(uExitVector)
5483	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5484	\|\| uIdtVector == X86_XCPT_PF))
5485	{
5486	enmReflect = VMXREFLECTXCPT_DF;
5487	}
5488	else if (uIdtVector == X86_XCPT_DF)
5489	enmReflect = VMXREFLECTXCPT_TF;
5490	}
5491	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5492	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5493	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5494	{
5495	/*
5496	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5497	* (whatever they are) as they reoccur when restarting the instruction.
5498	*/
5499	enmReflect = VMXREFLECTXCPT_XCPT;
5500	}
5501	}
5502	else
5503	{
5504	/*
5505	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5506	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5507	* original exception to the guest after handling the VM-exit.
5508	*/
5509	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5510	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5511	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5512	{
5513	enmReflect = VMXREFLECTXCPT_XCPT;
5514	}
5515	}
5516
5517	switch (enmReflect)
5518	{
5519	case VMXREFLECTXCPT_XCPT:
5520	{
5521	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5522	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5523	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5524
5525	uint32_t u32ErrCode = 0;
5526	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5527	{
5528	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5529	AssertRCReturn(rc, rc);
5530	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5531	}
5532
5533	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5534	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5535	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5536	rc = VINF_SUCCESS;
5537	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5538	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5539
5540	break;
5541	}
5542
5543	case VMXREFLECTXCPT_DF:
5544	{
5545	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5546	rc = VINF_HM_DOUBLE_FAULT;
5547	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5548	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5549
5550	break;
5551	}
5552
5553	case VMXREFLECTXCPT_TF:
5554	{
5555	rc = VINF_EM_RESET;
5556	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5557	uExitVector));
5558	break;
5559	}
5560
5561	default:
5562	Assert(rc == VINF_SUCCESS);
5563	break;
5564	}
5565	}
5566	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5567	return rc;
5568	}
5569
5570
5571	/**
5572	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5573	*
5574	* @returns VBox status code.
5575	* @param pVCpu Pointer to the VMCPU.
5576	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5577	* out-of-sync. Make sure to update the required fields
5578	* before using them.
5579	*
5580	* @remarks No-long-jump zone!!!
5581	*/
5582	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5583	{
5584	NOREF(pMixedCtx);
5585
5586	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5587	{
5588	uint32_t uVal = 0;
5589	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5590	AssertRCReturn(rc, rc);
5591
5592	uint32_t uShadow = 0;
5593	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5594	AssertRCReturn(rc, rc);
5595
5596	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5597	CPUMSetGuestCR0(pVCpu, uVal);
5598	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5599	}
5600	return VINF_SUCCESS;
5601	}
5602
5603
5604	/**
5605	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5606	*
5607	* @returns VBox status code.
5608	* @param pVCpu Pointer to the VMCPU.
5609	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5610	* out-of-sync. Make sure to update the required fields
5611	* before using them.
5612	*
5613	* @remarks No-long-jump zone!!!
5614	*/
5615	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5616	{
5617	NOREF(pMixedCtx);
5618
5619	int rc = VINF_SUCCESS;
5620	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5621	{
5622	uint32_t uVal = 0;
5623	uint32_t uShadow = 0;
5624	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5625	AssertRCReturn(rc, rc);
5626	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5627	AssertRCReturn(rc, rc);
5628
5629	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5630	CPUMSetGuestCR4(pVCpu, uVal);
5631	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5632	}
5633	return rc;
5634	}
5635
5636
5637	/**
5638	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5639	*
5640	* @returns VBox status code.
5641	* @param pVCpu Pointer to the VMCPU.
5642	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5643	* out-of-sync. Make sure to update the required fields
5644	* before using them.
5645	*
5646	* @remarks No-long-jump zone!!!
5647	*/
5648	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5649	{
5650	int rc = VINF_SUCCESS;
5651	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5652	{
5653	uint64_t u64Val = 0;
5654	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5655	AssertRCReturn(rc, rc);
5656
5657	pMixedCtx->rip = u64Val;
5658	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5659	}
5660	return rc;
5661	}
5662
5663
5664	/**
5665	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5666	*
5667	* @returns VBox status code.
5668	* @param pVCpu Pointer to the VMCPU.
5669	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5670	* out-of-sync. Make sure to update the required fields
5671	* before using them.
5672	*
5673	* @remarks No-long-jump zone!!!
5674	*/
5675	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5676	{
5677	int rc = VINF_SUCCESS;
5678	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5679	{
5680	uint64_t u64Val = 0;
5681	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5682	AssertRCReturn(rc, rc);
5683
5684	pMixedCtx->rsp = u64Val;
5685	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5686	}
5687	return rc;
5688	}
5689
5690
5691	/**
5692	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5693	*
5694	* @returns VBox status code.
5695	* @param pVCpu Pointer to the VMCPU.
5696	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5697	* out-of-sync. Make sure to update the required fields
5698	* before using them.
5699	*
5700	* @remarks No-long-jump zone!!!
5701	*/
5702	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5703	{
5704	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5705	{
5706	uint32_t uVal = 0;
5707	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5708	AssertRCReturn(rc, rc);
5709
5710	pMixedCtx->eflags.u32 = uVal;
5711	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5712	{
5713	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5714	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5715
5716	pMixedCtx->eflags.Bits.u1VM = 0;
5717	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5718	}
5719
5720	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5721	}
5722	return VINF_SUCCESS;
5723	}
5724
5725
5726	/**
5727	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5728	* guest-CPU context.
5729	*/
5730	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5731	{
5732	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5733	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5734	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5735	return rc;
5736	}
5737
5738
5739	/**
5740	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5741	* from the guest-state area in the VMCS.
5742	*
5743	* @param pVCpu Pointer to the VMCPU.
5744	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5745	* out-of-sync. Make sure to update the required fields
5746	* before using them.
5747	*
5748	* @remarks No-long-jump zone!!!
5749	*/
5750	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5751	{
5752	uint32_t uIntrState = 0;
5753	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5754	AssertRC(rc);
5755
5756	if (!uIntrState)
5757	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5758	else
5759	{
5760	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5761	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5762	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5763	AssertRC(rc);
5764	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5765	AssertRC(rc);
5766
5767	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5768	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5769	}
5770	}
5771
5772
5773	/**
5774	* Saves the guest's activity state.
5775	*
5776	* @returns VBox status code.
5777	* @param pVCpu Pointer to the VMCPU.
5778	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5779	* out-of-sync. Make sure to update the required fields
5780	* before using them.
5781	*
5782	* @remarks No-long-jump zone!!!
5783	*/
5784	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5785	{
5786	NOREF(pMixedCtx);
5787	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5788	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5789	return VINF_SUCCESS;
5790	}
5791
5792
5793	/**
5794	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5795	* the current VMCS into the guest-CPU context.
5796	*
5797	* @returns VBox status code.
5798	* @param pVCpu Pointer to the VMCPU.
5799	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5800	* out-of-sync. Make sure to update the required fields
5801	* before using them.
5802	*
5803	* @remarks No-long-jump zone!!!
5804	*/
5805	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5806	{
5807	int rc = VINF_SUCCESS;
5808	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5809	{
5810	uint32_t u32Val = 0;
5811	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5812	pMixedCtx->SysEnter.cs = u32Val;
5813	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5814	}
5815
5816	uint64_t u64Val = 0;
5817	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5818	{
5819	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5820	pMixedCtx->SysEnter.eip = u64Val;
5821	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5822	}
5823	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5824	{
5825	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5826	pMixedCtx->SysEnter.esp = u64Val;
5827	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5828	}
5829	return rc;
5830	}
5831
5832
5833	/**
5834	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5835	* the CPU back into the guest-CPU context.
5836	*
5837	* @returns VBox status code.
5838	* @param pVCpu Pointer to the VMCPU.
5839	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5840	* out-of-sync. Make sure to update the required fields
5841	* before using them.
5842	*
5843	* @remarks No-long-jump zone!!!
5844	*/
5845	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5846	{
5847	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5848	if ( HMVMX_IS_64BIT_HOST_MODE()
5849	&& pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5850	{
5851	/* We should not get preempted to a different CPU at this point while reading the MSRs. */
5852	VMMRZCallRing3Disable(pVCpu);
5853	HM_DISABLE_PREEMPT_IF_NEEDED();
5854
5855	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5856	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LAZY_MSRS))
5857	{
5858	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5859	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LAZY_MSRS;
5860	}
5861
5862	HM_RESTORE_PREEMPT_IF_NEEDED();
5863	VMMRZCallRing3Enable(pVCpu);
5864	}
5865	else
5866	{
5867	/* Darwin 32-bit/PAE kernel or 64-bit host running 32-bit guest. */
5868	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LAZY_MSRS;
5869	}
5870	#else /* HC_ARCH_BITS == 32 */
5871	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LAZY_MSRS;
5872	#endif
5873
5874	return VINF_SUCCESS;
5875	}
5876
5877
5878	/**
5879	* Saves the auto load/store'd guest MSRs from the current VMCS into
5880	* the guest-CPU context.
5881	*
5882	* @returns VBox status code.
5883	* @param pVCpu Pointer to the VMCPU.
5884	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5885	* out-of-sync. Make sure to update the required fields
5886	* before using them.
5887	*
5888	* @remarks No-long-jump zone!!!
5889	*/
5890	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5891	{
5892	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5893	return VINF_SUCCESS;
5894
5895	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5896	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5897	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5898	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5899	{
5900	switch (pMsr->u32Msr)
5901	{
5902	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5903	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5904	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5905	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5906	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5907	default:
5908	{
5909	AssertFailed();
5910	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5911	}
5912	}
5913	}
5914
5915	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5916	return VINF_SUCCESS;
5917	}
5918
5919
5920	/**
5921	* Saves the guest control registers from the current VMCS into the guest-CPU
5922	* context.
5923	*
5924	* @returns VBox status code.
5925	* @param pVCpu Pointer to the VMCPU.
5926	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5927	* out-of-sync. Make sure to update the required fields
5928	* before using them.
5929	*
5930	* @remarks No-long-jump zone!!!
5931	*/
5932	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5933	{
5934	/* Guest CR0. Guest FPU. */
5935	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5936	AssertRCReturn(rc, rc);
5937
5938	/* Guest CR4. */
5939	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5940	AssertRCReturn(rc, rc);
5941
5942	/* Guest CR2 - updated always during the world-switch or in #PF. */
5943	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5944	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5945	{
5946	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5947	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5948
5949	PVM pVM = pVCpu->CTX_SUFF(pVM);
5950	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5951	\|\| ( pVM->hm.s.fNestedPaging
5952	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5953	{
5954	uint64_t u64Val = 0;
5955	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5956	if (pMixedCtx->cr3 != u64Val)
5957	{
5958	CPUMSetGuestCR3(pVCpu, u64Val);
5959	if (VMMRZCallRing3IsEnabled(pVCpu))
5960	{
5961	PGMUpdateCR3(pVCpu, u64Val);
5962	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5963	}
5964	else
5965	{
5966	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5967	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5968	}
5969	}
5970
5971	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5972	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5973	{
5974	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5975	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5976	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5977	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5978
5979	if (VMMRZCallRing3IsEnabled(pVCpu))
5980	{
5981	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5982	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5983	}
5984	else
5985	{
5986	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5987	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5988	}
5989	}
5990	}
5991
5992	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5993	}
5994
5995	/*
5996	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5997	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5998	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5999	*
6000	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6001	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6002	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6003	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6004	*
6005	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6006	*/
6007	if (VMMRZCallRing3IsEnabled(pVCpu))
6008	{
6009	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6010	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6011
6012	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6013	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6014
6015	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6016	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6017	}
6018
6019	return rc;
6020	}
6021
6022
6023	/**
6024	* Reads a guest segment register from the current VMCS into the guest-CPU
6025	* context.
6026	*
6027	* @returns VBox status code.
6028	* @param pVCpu Pointer to the VMCPU.
6029	* @param idxSel Index of the selector in the VMCS.
6030	* @param idxLimit Index of the segment limit in the VMCS.
6031	* @param idxBase Index of the segment base in the VMCS.
6032	* @param idxAccess Index of the access rights of the segment in the VMCS.
6033	* @param pSelReg Pointer to the segment selector.
6034	*
6035	* @remarks No-long-jump zone!!!
6036	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6037	* macro as that takes care of whether to read from the VMCS cache or
6038	* not.
6039	*/
6040	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6041	PCPUMSELREG pSelReg)
6042	{
6043	NOREF(pVCpu);
6044
6045	uint32_t u32Val = 0;
6046	int rc = VMXReadVmcs32(idxSel, &u32Val);
6047	AssertRCReturn(rc, rc);
6048	pSelReg->Sel = (uint16_t)u32Val;
6049	pSelReg->ValidSel = (uint16_t)u32Val;
6050	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6051
6052	rc = VMXReadVmcs32(idxLimit, &u32Val);
6053	AssertRCReturn(rc, rc);
6054	pSelReg->u32Limit = u32Val;
6055
6056	uint64_t u64Val = 0;
6057	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6058	AssertRCReturn(rc, rc);
6059	pSelReg->u64Base = u64Val;
6060
6061	rc = VMXReadVmcs32(idxAccess, &u32Val);
6062	AssertRCReturn(rc, rc);
6063	pSelReg->Attr.u = u32Val;
6064
6065	/*
6066	* If VT-x marks the segment as unusable, most other bits remain undefined:
6067	* - For CS the L, D and G bits have meaning.
6068	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6069	* - For the remaining data segments no bits are defined.
6070	*
6071	* The present bit and the unusable bit has been observed to be set at the
6072	* same time (the selector was supposed to invalid as we started executing
6073	* a V8086 interrupt in ring-0).
6074	*
6075	* What should be important for the rest of the VBox code that the P bit is
6076	* cleared. Some of the other VBox code recognizes the unusable bit, but
6077	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6078	* safe side here, we'll strip off P and other bits we don't care about. If
6079	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6080	*
6081	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6082	*/
6083	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6084	{
6085	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6086
6087	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6088	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6089	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6090
6091	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6092	#ifdef DEBUG_bird
6093	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6094	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6095	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6096	#endif
6097	}
6098	return VINF_SUCCESS;
6099	}
6100
6101
6102	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6103	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6104	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6105	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6106	#else
6107	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6108	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6109	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6110	#endif
6111
6112
6113	/**
6114	* Saves the guest segment registers from the current VMCS into the guest-CPU
6115	* context.
6116	*
6117	* @returns VBox status code.
6118	* @param pVCpu Pointer to the VMCPU.
6119	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6120	* out-of-sync. Make sure to update the required fields
6121	* before using them.
6122	*
6123	* @remarks No-long-jump zone!!!
6124	*/
6125	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6126	{
6127	/* Guest segment registers. */
6128	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6129	{
6130	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6131	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6132	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6133	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6134	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6135	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6136	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6137
6138	/* Restore segment attributes for real-on-v86 mode hack. */
6139	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6140	{
6141	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6142	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6143	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6144	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6145	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6146	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6147	}
6148	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
6149	}
6150
6151	return VINF_SUCCESS;
6152	}
6153
6154
6155	/**
6156	* Saves the guest descriptor table registers and task register from the current
6157	* VMCS into the guest-CPU context.
6158	*
6159	* @returns VBox status code.
6160	* @param pVCpu Pointer to the VMCPU.
6161	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6162	* out-of-sync. Make sure to update the required fields
6163	* before using them.
6164	*
6165	* @remarks No-long-jump zone!!!
6166	*/
6167	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6168	{
6169	int rc = VINF_SUCCESS;
6170
6171	/* Guest LDTR. */
6172	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
6173	{
6174	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6175	AssertRCReturn(rc, rc);
6176	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
6177	}
6178
6179	/* Guest GDTR. */
6180	uint64_t u64Val = 0;
6181	uint32_t u32Val = 0;
6182	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
6183	{
6184	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6185	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6186	pMixedCtx->gdtr.pGdt = u64Val;
6187	pMixedCtx->gdtr.cbGdt = u32Val;
6188	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
6189	}
6190
6191	/* Guest IDTR. */
6192	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
6193	{
6194	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6195	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6196	pMixedCtx->idtr.pIdt = u64Val;
6197	pMixedCtx->idtr.cbIdt = u32Val;
6198	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
6199	}
6200
6201	/* Guest TR. */
6202	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
6203	{
6204	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6205	AssertRCReturn(rc, rc);
6206
6207	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6208	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6209	{
6210	rc = VMXLOCAL_READ_SEG(TR, tr);
6211	AssertRCReturn(rc, rc);
6212	}
6213	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
6214	}
6215	return rc;
6216	}
6217
6218	#undef VMXLOCAL_READ_SEG
6219
6220
6221	/**
6222	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6223	* context.
6224	*
6225	* @returns VBox status code.
6226	* @param pVCpu Pointer to the VMCPU.
6227	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6228	* out-of-sync. Make sure to update the required fields
6229	* before using them.
6230	*
6231	* @remarks No-long-jump zone!!!
6232	*/
6233	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6234	{
6235	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
6236	{
6237	if (!pVCpu->hm.s.fUsingHyperDR7)
6238	{
6239	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6240	uint32_t u32Val;
6241	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6242	pMixedCtx->dr[7] = u32Val;
6243	}
6244
6245	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
6246	}
6247	return VINF_SUCCESS;
6248	}
6249
6250
6251	/**
6252	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6253	*
6254	* @returns VBox status code.
6255	* @param pVCpu Pointer to the VMCPU.
6256	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6257	* out-of-sync. Make sure to update the required fields
6258	* before using them.
6259	*
6260	* @remarks No-long-jump zone!!!
6261	*/
6262	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6263	{
6264	NOREF(pMixedCtx);
6265
6266	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6267	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
6268	return VINF_SUCCESS;
6269	}
6270
6271
6272	/**
6273	* Saves the entire guest state from the currently active VMCS into the
6274	* guest-CPU context. This essentially VMREADs all guest-data.
6275	*
6276	* @returns VBox status code.
6277	* @param pVCpu Pointer to the VMCPU.
6278	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6279	* out-of-sync. Make sure to update the required fields
6280	* before using them.
6281	*/
6282	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6283	{
6284	Assert(pVCpu);
6285	Assert(pMixedCtx);
6286
6287	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
6288	return VINF_SUCCESS;
6289
6290	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6291	again on the ring-3 callback path, there is no real need to. */
6292	if (VMMRZCallRing3IsEnabled(pVCpu))
6293	VMMR0LogFlushDisable(pVCpu);
6294	else
6295	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6296	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6297
6298	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6299	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6300
6301	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6302	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6303
6304	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6305	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6306
6307	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6308	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6309
6310	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6311	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6312
6313	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6314	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6315
6316	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6317	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6318
6319	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6320	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6321
6322	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6323	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6324
6325	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6326	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6327
6328	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
6329	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
6330
6331	if (VMMRZCallRing3IsEnabled(pVCpu))
6332	VMMR0LogFlushEnable(pVCpu);
6333
6334	return rc;
6335	}
6336
6337
6338	/**
6339	* Check per-VM and per-VCPU force flag actions that require us to go back to
6340	* ring-3 for one reason or another.
6341	*
6342	* @returns VBox status code (information status code included).
6343	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6344	* ring-3.
6345	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6346	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6347	* interrupts)
6348	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6349	* all EMTs to be in ring-3.
6350	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6351	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6352	* to the EM loop.
6353	*
6354	* @param pVM Pointer to the VM.
6355	* @param pVCpu Pointer to the VMCPU.
6356	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6357	* out-of-sync. Make sure to update the required fields
6358	* before using them.
6359	*/
6360	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6361	{
6362	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6363
6364	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6365	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6366	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6367	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6368	{
6369	/* We need the control registers now, make sure the guest-CPU context is updated. */
6370	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6371	AssertRCReturn(rc3, rc3);
6372
6373	/* Pending HM CR3 sync. */
6374	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6375	{
6376	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6377	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6378	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6379	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6380	}
6381
6382	/* Pending HM PAE PDPEs. */
6383	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6384	{
6385	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6386	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6387	}
6388
6389	/* Pending PGM C3 sync. */
6390	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6391	{
6392	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6393	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6394	if (rc2 != VINF_SUCCESS)
6395	{
6396	AssertRC(rc2);
6397	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6398	return rc2;
6399	}
6400	}
6401
6402	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6403	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6404	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6405	{
6406	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6407	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6408	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6409	return rc2;
6410	}
6411
6412	/* Pending VM request packets, such as hardware interrupts. */
6413	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6414	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6415	{
6416	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6417	return VINF_EM_PENDING_REQUEST;
6418	}
6419
6420	/* Pending PGM pool flushes. */
6421	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6422	{
6423	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6424	return VINF_PGM_POOL_FLUSH_PENDING;
6425	}
6426
6427	/* Pending DMA requests. */
6428	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6429	{
6430	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6431	return VINF_EM_RAW_TO_R3;
6432	}
6433	}
6434
6435	return VINF_SUCCESS;
6436	}
6437
6438
6439	/**
6440	* Converts any TRPM trap into a pending HM event. This is typically used when
6441	* entering from ring-3 (not longjmp returns).
6442	*
6443	* @param pVCpu Pointer to the VMCPU.
6444	*/
6445	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6446	{
6447	Assert(TRPMHasTrap(pVCpu));
6448	Assert(!pVCpu->hm.s.Event.fPending);
6449
6450	uint8_t uVector;
6451	TRPMEVENT enmTrpmEvent;
6452	RTGCUINT uErrCode;
6453	RTGCUINTPTR GCPtrFaultAddress;
6454	uint8_t cbInstr;
6455
6456	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6457	AssertRC(rc);
6458
6459	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6460	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6461	if (enmTrpmEvent == TRPM_TRAP)
6462	{
6463	switch (uVector)
6464	{
6465	case X86_XCPT_BP:
6466	case X86_XCPT_OF:
6467	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6468	break;
6469
6470	case X86_XCPT_PF:
6471	case X86_XCPT_DF:
6472	case X86_XCPT_TS:
6473	case X86_XCPT_NP:
6474	case X86_XCPT_SS:
6475	case X86_XCPT_GP:
6476	case X86_XCPT_AC:
6477	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6478	/* no break! */
6479	default:
6480	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6481	break;
6482	}
6483	}
6484	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6485	{
6486	if (uVector == X86_XCPT_NMI)
6487	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6488	else
6489	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6490	}
6491	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6492	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6493	else
6494	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6495
6496	rc = TRPMResetTrap(pVCpu);
6497	AssertRC(rc);
6498	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6499	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6500
6501	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6502	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6503	}
6504
6505
6506	/**
6507	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6508	* VT-x to execute any instruction.
6509	*
6510	* @param pvCpu Pointer to the VMCPU.
6511	*/
6512	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6513	{
6514	Assert(pVCpu->hm.s.Event.fPending);
6515
6516	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6517	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6518	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6519	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6520
6521	/* If a trap was already pending, we did something wrong! */
6522	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6523
6524	TRPMEVENT enmTrapType;
6525	switch (uVectorType)
6526	{
6527	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6528	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6529	enmTrapType = TRPM_HARDWARE_INT;
6530	break;
6531
6532	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6533	enmTrapType = TRPM_SOFTWARE_INT;
6534	break;
6535
6536	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6537	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6538	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6539	enmTrapType = TRPM_TRAP;
6540	break;
6541
6542	default:
6543	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6544	enmTrapType = TRPM_32BIT_HACK;
6545	break;
6546	}
6547
6548	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6549
6550	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6551	AssertRC(rc);
6552
6553	if (fErrorCodeValid)
6554	TRPMSetErrorCode(pVCpu, uErrorCode);
6555
6556	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6557	&& uVector == X86_XCPT_PF)
6558	{
6559	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6560	}
6561	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6562	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6563	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6564	{
6565	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6566	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6567	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6568	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6569	}
6570	pVCpu->hm.s.Event.fPending = false;
6571	}
6572
6573
6574	/**
6575	* Does the necessary state syncing before returning to ring-3 for any reason
6576	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6577	*
6578	* @returns VBox status code.
6579	* @param pVM Pointer to the VM.
6580	* @param pVCpu Pointer to the VMCPU.
6581	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6582	* be out-of-sync. Make sure to update the required
6583	* fields before using them.
6584	* @param fSaveGuestState Whether to save the guest state or not.
6585	*
6586	* @remarks If you modify code here, make sure to check whether
6587	* hmR0VmxCallRing3Callback() needs to be updated too!!!
6588	* @remarks No-long-jmp zone!!!
6589	*/
6590	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6591	{
6592	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6593	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6594
6595	RTCPUID idCpu = RTMpCpuId();
6596	Log4Func(("HostCpuId=%u\n", idCpu));
6597
6598	/* Save the guest state if necessary. */
6599	if ( fSaveGuestState
6600	&& pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6601	{
6602	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6603	AssertRCReturn(rc, rc);
6604	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6605	}
6606
6607	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6608	if (CPUMIsGuestFPUStateActive(pVCpu))
6609	{
6610	/* We shouldn't reload CR0 without saving it first. */
6611	if (!fSaveGuestState)
6612	{
6613	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6614	AssertRCReturn(rc, rc);
6615	}
6616	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6617	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6618	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6619	}
6620
6621	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6622	#ifdef VBOX_STRICT
6623	if (CPUMIsHyperDebugStateActive(pVCpu))
6624	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6625	#endif
6626	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6627	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6628	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6629	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6630
6631	#if HC_ARCH_BITS == 64
6632	/* Restore host-state bits that VT-x only restores partially. */
6633	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6634	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6635	{
6636	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6637	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6638	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6639	}
6640	#endif
6641
6642	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
6643	/* Restore the host MSRs as we're leaving VT-x context. */
6644	if ( HMVMX_IS_64BIT_HOST_MODE()
6645	&& pVM->hm.s.fAllow64BitGuests
6646	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6647	{
6648	/* We shouldn't reload the guest MSRs without saving it first. */
6649	if (!fSaveGuestState)
6650	{
6651	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6652	AssertRCReturn(rc, rc);
6653	}
6654	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LAZY_MSRS);
6655	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6656	Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6657	}
6658	#endif
6659
6660	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6661	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6662	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6663	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6664	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6665	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6666	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6667	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6668
6669	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6670
6671	/** @todo This partially defeats the purpose of having preemption hooks.
6672	* The problem is, deregistering the hooks should be moved to a place that
6673	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6674	* context.
6675	*/
6676	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6677	{
6678	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6679	AssertRCReturn(rc, rc);
6680
6681	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6682	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6683	}
6684	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6685	NOREF(idCpu);
6686
6687	return VINF_SUCCESS;
6688	}
6689
6690
6691	/**
6692	* Leaves the VT-x session.
6693	*
6694	* @returns VBox status code.
6695	* @param pVM Pointer to the VM.
6696	* @param pVCpu Pointer to the VMCPU.
6697	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6698	* out-of-sync. Make sure to update the required fields
6699	* before using them.
6700	*
6701	* @remarks No-long-jmp zone!!!
6702	*/
6703	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6704	{
6705	HM_DISABLE_PREEMPT_IF_NEEDED();
6706	HMVMX_ASSERT_CPU_SAFE();
6707	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6708	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6709
6710	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6711	and done this from the VMXR0ThreadCtxCallback(). */
6712	if (!pVCpu->hm.s.fLeaveDone)
6713	{
6714	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6715	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6716	pVCpu->hm.s.fLeaveDone = true;
6717	}
6718
6719	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6720	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6721	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6722	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6723	VMMR0ThreadCtxHooksDeregister(pVCpu);
6724
6725	/* Leave HM context. This takes care of local init (term). */
6726	int rc = HMR0LeaveCpu(pVCpu);
6727
6728	HM_RESTORE_PREEMPT_IF_NEEDED();
6729
6730	return rc;
6731	}
6732
6733
6734	/**
6735	* Does the necessary state syncing before doing a longjmp to ring-3.
6736	*
6737	* @returns VBox status code.
6738	* @param pVM Pointer to the VM.
6739	* @param pVCpu Pointer to the VMCPU.
6740	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6741	* out-of-sync. Make sure to update the required fields
6742	* before using them.
6743	*
6744	* @remarks No-long-jmp zone!!!
6745	*/
6746	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6747	{
6748	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6749	}
6750
6751
6752	/**
6753	* Take necessary actions before going back to ring-3.
6754	*
6755	* An action requires us to go back to ring-3. This function does the necessary
6756	* steps before we can safely return to ring-3. This is not the same as longjmps
6757	* to ring-3, this is voluntary and prepares the guest so it may continue
6758	* executing outside HM (recompiler/IEM).
6759	*
6760	* @returns VBox status code.
6761	* @param pVM Pointer to the VM.
6762	* @param pVCpu Pointer to the VMCPU.
6763	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6764	* out-of-sync. Make sure to update the required fields
6765	* before using them.
6766	* @param rcExit The reason for exiting to ring-3. Can be
6767	* VINF_VMM_UNKNOWN_RING3_CALL.
6768	*/
6769	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6770	{
6771	Assert(pVM);
6772	Assert(pVCpu);
6773	Assert(pMixedCtx);
6774	HMVMX_ASSERT_PREEMPT_SAFE();
6775
6776	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6777	{
6778	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6779	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6780	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6781	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6782	}
6783
6784	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6785	VMMRZCallRing3Disable(pVCpu);
6786	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6787
6788	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6789	if (pVCpu->hm.s.Event.fPending)
6790	{
6791	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6792	Assert(!pVCpu->hm.s.Event.fPending);
6793	}
6794
6795	/* Save guest state and restore host state bits. */
6796	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6797	AssertRCReturn(rc, rc);
6798	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6799
6800	/* Sync recompiler state. */
6801	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6802	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6803	\| CPUM_CHANGED_LDTR
6804	\| CPUM_CHANGED_GDTR
6805	\| CPUM_CHANGED_IDTR
6806	\| CPUM_CHANGED_TR
6807	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6808	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6809	if ( pVM->hm.s.fNestedPaging
6810	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6811	{
6812	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6813	}
6814
6815	Assert(!pVCpu->hm.s.fClearTrapFlag);
6816
6817	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6818	if (rcExit != VINF_EM_RAW_INTERRUPT)
6819	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6820
6821	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6822
6823	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6824	VMMRZCallRing3RemoveNotification(pVCpu);
6825	VMMRZCallRing3Enable(pVCpu);
6826
6827	return rc;
6828	}
6829
6830
6831	/**
6832	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6833	* longjump to ring-3 and possibly get preempted.
6834	*
6835	* @returns VBox status code.
6836	* @param pVCpu Pointer to the VMCPU.
6837	* @param enmOperation The operation causing the ring-3 longjump.
6838	* @param pvUser Opaque pointer to the guest-CPU context. The data
6839	* may be out-of-sync. Make sure to update the required
6840	* fields before using them.
6841	*
6842	* @remarks If you modify code here, make sure to check whether
6843	* hmR0VmxLeave() needs to be updated too!!!
6844	*/
6845	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6846	{
6847	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6848	{
6849	/* If anything here asserts or fails, good luck. */
6850	VMMRZCallRing3RemoveNotification(pVCpu);
6851	VMMRZCallRing3Disable(pVCpu);
6852	HM_DISABLE_PREEMPT_IF_NEEDED();
6853
6854	PVM pVM = pVCpu->CTX_SUFF(pVM);
6855	if (CPUMIsGuestFPUStateActive(pVCpu))
6856	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6857
6858	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6859
6860	#if HC_ARCH_BITS == 64
6861	/* Restore host-state bits that VT-x only restores partially. */
6862	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6863	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6864	{
6865	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6866	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6867	}
6868	#endif
6869
6870	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
6871	/* Restore the host MSRs as we're leaving VT-x context. */
6872	if ( HMVMX_IS_64BIT_HOST_MODE()
6873	&& pVM->hm.s.fAllow64BitGuests
6874	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6875	{
6876	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6877	}
6878	#endif
6879	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6880	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6881	{
6882	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6883	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6884	}
6885
6886	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6887	VMMR0ThreadCtxHooksDeregister(pVCpu);
6888
6889	HMR0LeaveCpu(pVCpu);
6890	HM_RESTORE_PREEMPT_IF_NEEDED();
6891	return VINF_SUCCESS;
6892	}
6893
6894	Assert(pVCpu);
6895	Assert(pvUser);
6896	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6897	HMVMX_ASSERT_PREEMPT_SAFE();
6898
6899	VMMRZCallRing3Disable(pVCpu);
6900	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6901
6902	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6903	enmOperation));
6904
6905	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6906	AssertRCReturn(rc, rc);
6907
6908	VMMRZCallRing3Enable(pVCpu);
6909	return VINF_SUCCESS;
6910	}
6911
6912
6913	/**
6914	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6915	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6916	*
6917	* @param pVCpu Pointer to the VMCPU.
6918	*/
6919	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6920	{
6921	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6922	{
6923	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6924	{
6925	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6926	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6927	AssertRC(rc);
6928	}
6929	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6930	}
6931
6932
6933	/**
6934	* Evaluates the event to be delivered to the guest and sets it as the pending
6935	* event.
6936	*
6937	* @param pVCpu Pointer to the VMCPU.
6938	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6939	* out-of-sync. Make sure to update the required fields
6940	* before using them.
6941	*/
6942	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6943	{
6944	Assert(!pVCpu->hm.s.Event.fPending);
6945
6946	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6947	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6948	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6949	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6950
6951	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6952	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6953	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6954	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6955	Assert(!TRPMHasTrap(pVCpu));
6956
6957	/** @todo SMI. SMIs take priority over NMIs. */
6958	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6959	{
6960	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6961	if ( !fBlockMovSS
6962	&& !fBlockSti)
6963	{
6964	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6965	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6966	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6967	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6968
6969	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6970	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6971	}
6972	else
6973	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6974	}
6975	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6976	&& !pVCpu->hm.s.fSingleInstruction)
6977	{
6978	/*
6979	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6980	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6981	* evaluated here and not set as pending, solely based on the force-flags.
6982	*/
6983	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6984	AssertRC(rc);
6985	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6986	if ( !fBlockInt
6987	&& !fBlockSti
6988	&& !fBlockMovSS)
6989	{
6990	uint8_t u8Interrupt;
6991	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6992	if (RT_SUCCESS(rc))
6993	{
6994	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6995	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6996	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6997
6998	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
6999	}
7000	else
7001	{
7002	/** @todo Does this actually happen? If not turn it into an assertion. */
7003	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7004	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7005	}
7006	}
7007	else
7008	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7009	}
7010	}
7011
7012
7013	/**
7014	* Injects any pending events into the guest if the guest is in a state to
7015	* receive them.
7016	*
7017	* @returns VBox status code (informational status codes included).
7018	* @param pVCpu Pointer to the VMCPU.
7019	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7020	* out-of-sync. Make sure to update the required fields
7021	* before using them.
7022	*/
7023	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7024	{
7025	HMVMX_ASSERT_PREEMPT_SAFE();
7026	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7027
7028	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7029	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7030	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7031	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7032
7033	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
7034	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7035	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7036	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7037	Assert(!TRPMHasTrap(pVCpu));
7038
7039	int rc = VINF_SUCCESS;
7040	if (pVCpu->hm.s.Event.fPending)
7041	{
7042	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
7043	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7044	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7045	{
7046	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7047	AssertRCReturn(rc, rc);
7048	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7049	Assert(!fBlockInt);
7050	Assert(!fBlockSti);
7051	Assert(!fBlockMovSS);
7052	}
7053	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7054	{
7055	Assert(!fBlockSti);
7056	Assert(!fBlockMovSS);
7057	}
7058	#endif
7059	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
7060	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7061	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7062	AssertRCReturn(rc, rc);
7063
7064	/* Update the interruptibility-state as it could have been changed by
7065	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7066	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7067	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7068
7069	#ifdef VBOX_WITH_STATISTICS
7070	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7071	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7072	else
7073	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7074	#endif
7075	}
7076
7077	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7078	if ( !pVCpu->hm.s.fSingleInstruction
7079	&& !DBGFIsStepping(pVCpu))
7080	{
7081	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7082	AssertRCReturn(rc2, rc2);
7083	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7084	{
7085	/*
7086	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7087	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7088	* See Intel spec. 27.3.4 "Saving Non-Register State".
7089	*/
7090	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7091	AssertRCReturn(rc2, rc2);
7092	}
7093	}
7094	else
7095	{
7096	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
7097	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
7098	uIntrState = 0;
7099	}
7100
7101	/*
7102	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7103	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7104	*/
7105	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7106	AssertRC(rc2);
7107
7108	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7109	NOREF(fBlockMovSS); NOREF(fBlockSti);
7110	return rc;
7111	}
7112
7113
7114	/**
7115	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7116	*
7117	* @param pVCpu Pointer to the VMCPU.
7118	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7119	* out-of-sync. Make sure to update the required fields
7120	* before using them.
7121	*/
7122	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7123	{
7124	NOREF(pMixedCtx);
7125	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7126	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7127	}
7128
7129
7130	/**
7131	* Injects a double-fault (#DF) exception into the VM.
7132	*
7133	* @returns VBox status code (informational status code included).
7134	* @param pVCpu Pointer to the VMCPU.
7135	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7136	* out-of-sync. Make sure to update the required fields
7137	* before using them.
7138	*/
7139	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7140	{
7141	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7142	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7143	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7144	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7145	puIntrState);
7146	}
7147
7148
7149	/**
7150	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7151	*
7152	* @param pVCpu Pointer to the VMCPU.
7153	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7154	* out-of-sync. Make sure to update the required fields
7155	* before using them.
7156	*/
7157	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7158	{
7159	NOREF(pMixedCtx);
7160	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7161	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7162	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7163	}
7164
7165
7166	/**
7167	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7168	*
7169	* @param pVCpu Pointer to the VMCPU.
7170	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7171	* out-of-sync. Make sure to update the required fields
7172	* before using them.
7173	* @param cbInstr The value of RIP that is to be pushed on the guest
7174	* stack.
7175	*/
7176	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7177	{
7178	NOREF(pMixedCtx);
7179	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7180	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7181	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7182	}
7183
7184
7185	/**
7186	* Injects a general-protection (#GP) fault into the VM.
7187	*
7188	* @returns VBox status code (informational status code included).
7189	* @param pVCpu Pointer to the VMCPU.
7190	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7191	* out-of-sync. Make sure to update the required fields
7192	* before using them.
7193	* @param u32ErrorCode The error code associated with the #GP.
7194	*/
7195	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7196	uint32_t *puIntrState)
7197	{
7198	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7199	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7200	if (fErrorCodeValid)
7201	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7202	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7203	puIntrState);
7204	}
7205
7206
7207	/**
7208	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7209	*
7210	* @param pVCpu Pointer to the VMCPU.
7211	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7212	* out-of-sync. Make sure to update the required fields
7213	* before using them.
7214	* @param uVector The software interrupt vector number.
7215	* @param cbInstr The value of RIP that is to be pushed on the guest
7216	* stack.
7217	*/
7218	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7219	{
7220	NOREF(pMixedCtx);
7221	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7222	if ( uVector == X86_XCPT_BP
7223	\|\| uVector == X86_XCPT_OF)
7224	{
7225	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7226	}
7227	else
7228	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7229	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7230	}
7231
7232
7233	/**
7234	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7235	* stack.
7236	*
7237	* @returns VBox status code (information status code included).
7238	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7239	* @param pVM Pointer to the VM.
7240	* @param pMixedCtx Pointer to the guest-CPU context.
7241	* @param uValue The value to push to the guest stack.
7242	*/
7243	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7244	{
7245	/*
7246	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7247	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7248	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7249	*/
7250	if (pMixedCtx->sp == 1)
7251	return VINF_EM_RESET;
7252	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7253	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7254	AssertRCReturn(rc, rc);
7255	return rc;
7256	}
7257
7258
7259	/**
7260	* Injects an event into the guest upon VM-entry by updating the relevant fields
7261	* in the VM-entry area in the VMCS.
7262	*
7263	* @returns VBox status code (informational error codes included).
7264	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7265	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7266	*
7267	* @param pVCpu Pointer to the VMCPU.
7268	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7269	* be out-of-sync. Make sure to update the required
7270	* fields before using them.
7271	* @param u64IntInfo The VM-entry interruption-information field.
7272	* @param cbInstr The VM-entry instruction length in bytes (for
7273	* software interrupts, exceptions and privileged
7274	* software exceptions).
7275	* @param u32ErrCode The VM-entry exception error code.
7276	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7277	* @param puIntrState Pointer to the current guest interruptibility-state.
7278	* This interruptibility-state will be updated if
7279	* necessary. This cannot not be NULL.
7280	*
7281	* @remarks Requires CR0!
7282	* @remarks No-long-jump zone!!!
7283	*/
7284	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7285	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7286	{
7287	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7288	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7289	Assert(puIntrState);
7290	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7291
7292	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7293	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7294
7295	#ifdef VBOX_STRICT
7296	/* Validate the error-code-valid bit for hardware exceptions. */
7297	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7298	{
7299	switch (uVector)
7300	{
7301	case X86_XCPT_PF:
7302	case X86_XCPT_DF:
7303	case X86_XCPT_TS:
7304	case X86_XCPT_NP:
7305	case X86_XCPT_SS:
7306	case X86_XCPT_GP:
7307	case X86_XCPT_AC:
7308	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7309	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7310	/* fallthru */
7311	default:
7312	break;
7313	}
7314	}
7315	#endif
7316
7317	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7318	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7319	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7320
7321	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7322
7323	/* We require CR0 to check if the guest is in real-mode. */
7324	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7325	AssertRCReturn(rc, rc);
7326
7327	/*
7328	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7329	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7330	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7331	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7332	*/
7333	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7334	{
7335	PVM pVM = pVCpu->CTX_SUFF(pVM);
7336	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7337	{
7338	Assert(PDMVmmDevHeapIsEnabled(pVM));
7339	Assert(pVM->hm.s.vmx.pRealModeTSS);
7340
7341	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7342	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7343	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7344	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7345	AssertRCReturn(rc, rc);
7346	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
7347
7348	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7349	const size_t cbIdtEntry = sizeof(X86IDTR16);
7350	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7351	{
7352	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7353	if (uVector == X86_XCPT_DF)
7354	return VINF_EM_RESET;
7355	else if (uVector == X86_XCPT_GP)
7356	{
7357	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7358	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7359	}
7360
7361	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7362	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7363	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7364	}
7365
7366	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7367	uint16_t uGuestIp = pMixedCtx->ip;
7368	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7369	{
7370	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7371	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7372	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7373	}
7374	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7375	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7376
7377	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7378	X86IDTR16 IdtEntry;
7379	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7380	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7381	AssertRCReturn(rc, rc);
7382
7383	/* Construct the stack frame for the interrupt/exception handler. */
7384	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7385	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7386	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7387	AssertRCReturn(rc, rc);
7388
7389	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7390	if (rc == VINF_SUCCESS)
7391	{
7392	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7393	pMixedCtx->rip = IdtEntry.offSel;
7394	pMixedCtx->cs.Sel = IdtEntry.uSel;
7395	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7396	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7397	&& uVector == X86_XCPT_PF)
7398	{
7399	pMixedCtx->cr2 = GCPtrFaultAddress;
7400	}
7401
7402	/* If any other guest-state bits are changed here, make sure to update
7403	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7404	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7405	\| HM_CHANGED_GUEST_RIP
7406	\| HM_CHANGED_GUEST_RFLAGS
7407	\| HM_CHANGED_GUEST_RSP);
7408
7409	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7410	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7411	{
7412	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7413	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7414	Log4(("Clearing inhibition due to STI.\n"));
7415	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7416	}
7417	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7418
7419	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7420	it, if we are returning to ring-3 before executing guest code. */
7421	pVCpu->hm.s.Event.fPending = false;
7422	}
7423	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7424	return rc;
7425	}
7426	else
7427	{
7428	/*
7429	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7430	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7431	*/
7432	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7433	}
7434	}
7435
7436	/* Validate. */
7437	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7438	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7439	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7440
7441	/* Inject. */
7442	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7443	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7444	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7445	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7446
7447	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7448	&& uVector == X86_XCPT_PF)
7449	{
7450	pMixedCtx->cr2 = GCPtrFaultAddress;
7451	}
7452
7453	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7454	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7455
7456	AssertRCReturn(rc, rc);
7457	return rc;
7458	}
7459
7460
7461	/**
7462	* Clears the interrupt-window exiting control in the VMCS and if necessary
7463	* clears the current event in the VMCS as well.
7464	*
7465	* @returns VBox status code.
7466	* @param pVCpu Pointer to the VMCPU.
7467	*
7468	* @remarks Use this function only to clear events that have not yet been
7469	* delivered to the guest but are injected in the VMCS!
7470	* @remarks No-long-jump zone!!!
7471	*/
7472	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7473	{
7474	int rc;
7475	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7476
7477	/* Clear interrupt-window exiting control. */
7478	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7479	{
7480	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7481	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7482	AssertRC(rc);
7483	}
7484
7485	if (!pVCpu->hm.s.Event.fPending)
7486	return;
7487
7488	#ifdef VBOX_STRICT
7489	uint32_t u32EntryInfo;
7490	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7491	AssertRC(rc);
7492	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7493	#endif
7494
7495	/* Clear the entry-interruption field (including the valid bit). */
7496	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7497	AssertRC(rc);
7498
7499	/* Clear the pending debug exception field. */
7500	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7501	AssertRC(rc);
7502	}
7503
7504
7505	/**
7506	* Enters the VT-x session.
7507	*
7508	* @returns VBox status code.
7509	* @param pVM Pointer to the VM.
7510	* @param pVCpu Pointer to the VMCPU.
7511	* @param pCpu Pointer to the CPU info struct.
7512	*/
7513	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7514	{
7515	AssertPtr(pVM);
7516	AssertPtr(pVCpu);
7517	Assert(pVM->hm.s.vmx.fSupported);
7518	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7519	NOREF(pCpu); NOREF(pVM);
7520
7521	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7522	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7523
7524	#ifdef VBOX_STRICT
7525	/* Make sure we're in VMX root mode. */
7526	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7527	if (!(u32HostCR4 & X86_CR4_VMXE))
7528	{
7529	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7530	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7531	}
7532	#endif
7533
7534	/*
7535	* Load the VCPU's VMCS as the current (and active) one.
7536	*/
7537	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7538	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7539	if (RT_FAILURE(rc))
7540	return rc;
7541
7542	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7543	pVCpu->hm.s.fLeaveDone = false;
7544	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7545
7546	return VINF_SUCCESS;
7547	}
7548
7549
7550	/**
7551	* The thread-context callback (only on platforms which support it).
7552	*
7553	* @param enmEvent The thread-context event.
7554	* @param pVCpu Pointer to the VMCPU.
7555	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7556	* @thread EMT(pVCpu)
7557	*/
7558	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7559	{
7560	NOREF(fGlobalInit);
7561
7562	switch (enmEvent)
7563	{
7564	case RTTHREADCTXEVENT_PREEMPTING:
7565	{
7566	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7567	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7568	VMCPU_ASSERT_EMT(pVCpu);
7569
7570	PVM pVM = pVCpu->CTX_SUFF(pVM);
7571	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7572
7573	/* No longjmps (logger flushes, locks) in this fragile context. */
7574	VMMRZCallRing3Disable(pVCpu);
7575	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7576
7577	/*
7578	* Restore host-state (FPU, debug etc.)
7579	*/
7580	if (!pVCpu->hm.s.fLeaveDone)
7581	{
7582	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7583	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7584	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7585	pVCpu->hm.s.fLeaveDone = true;
7586	}
7587
7588	/* Leave HM context, takes care of local init (term). */
7589	int rc = HMR0LeaveCpu(pVCpu);
7590	AssertRC(rc); NOREF(rc);
7591
7592	/* Restore longjmp state. */
7593	VMMRZCallRing3Enable(pVCpu);
7594	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7595	break;
7596	}
7597
7598	case RTTHREADCTXEVENT_RESUMED:
7599	{
7600	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7601	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7602	VMCPU_ASSERT_EMT(pVCpu);
7603
7604	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7605	VMMRZCallRing3Disable(pVCpu);
7606	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7607
7608	/* Initialize the bare minimum state required for HM. This takes care of
7609	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7610	int rc = HMR0EnterCpu(pVCpu);
7611	AssertRC(rc);
7612	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7613
7614	/* Load the active VMCS as the current one. */
7615	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7616	{
7617	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7618	AssertRC(rc); NOREF(rc);
7619	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7620	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7621	}
7622	pVCpu->hm.s.fLeaveDone = false;
7623
7624	/* Restore longjmp state. */
7625	VMMRZCallRing3Enable(pVCpu);
7626	break;
7627	}
7628
7629	default:
7630	break;
7631	}
7632	}
7633
7634
7635	/**
7636	* Saves the host state in the VMCS host-state.
7637	* Sets up the VM-exit MSR-load area.
7638	*
7639	* The CPU state will be loaded from these fields on every successful VM-exit.
7640	*
7641	* @returns VBox status code.
7642	* @param pVM Pointer to the VM.
7643	* @param pVCpu Pointer to the VMCPU.
7644	*
7645	* @remarks No-long-jump zone!!!
7646	*/
7647	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7648	{
7649	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7650
7651	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7652	return VINF_SUCCESS;
7653
7654	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7655	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7656
7657	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7658	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7659
7660	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7661	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7662
7663	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7664	return rc;
7665	}
7666
7667
7668	/**
7669	* Saves the host state in the VMCS host-state.
7670	*
7671	* @returns VBox status code.
7672	* @param pVM Pointer to the VM.
7673	* @param pVCpu Pointer to the VMCPU.
7674	*
7675	* @remarks No-long-jump zone!!!
7676	*/
7677	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7678	{
7679	AssertPtr(pVM);
7680	AssertPtr(pVCpu);
7681
7682	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7683
7684	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7685	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7686	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7687	return hmR0VmxSaveHostState(pVM, pVCpu);
7688	}
7689
7690
7691	/**
7692	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7693	* loaded from these fields on every successful VM-entry.
7694	*
7695	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7696	* Sets up the VM-entry controls.
7697	* Sets up the appropriate VMX non-root function to execute guest code based on
7698	* the guest CPU mode.
7699	*
7700	* @returns VBox status code.
7701	* @param pVM Pointer to the VM.
7702	* @param pVCpu Pointer to the VMCPU.
7703	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7704	* out-of-sync. Make sure to update the required fields
7705	* before using them.
7706	*
7707	* @remarks No-long-jump zone!!!
7708	*/
7709	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7710	{
7711	AssertPtr(pVM);
7712	AssertPtr(pVCpu);
7713	AssertPtr(pMixedCtx);
7714	HMVMX_ASSERT_PREEMPT_SAFE();
7715
7716	#ifdef LOG_ENABLED
7717	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7718	* probably not initialized yet? Anyway this will do for now.
7719	*
7720	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7721	* interface and disable ring-3 calls when thread-context hooks are not
7722	* available. */
7723	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7724	VMMR0LogFlushDisable(pVCpu);
7725	#endif
7726
7727	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7728
7729	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7730
7731	/* Determine real-on-v86 mode. */
7732	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7733	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7734	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7735	{
7736	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7737	}
7738
7739	/*
7740	* Load the guest-state into the VMCS.
7741	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7742	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7743	*/
7744	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7745	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7746
7747	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7748	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7749	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7750
7751	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7752	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7753	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7754
7755	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7756	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7757
7758	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7759	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7760
7761	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7762	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7763	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7764
7765	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7766	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7767
7768	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7769	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7770
7771	/*
7772	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7773	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7774	*/
7775	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7776	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7777
7778	/* Clear any unused and reserved bits. */
7779	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7780
7781	#ifdef LOG_ENABLED
7782	/* Only reenable log-flushing if the caller has it enabled. */
7783	if (!fCallerDisabledLogFlush)
7784	VMMR0LogFlushEnable(pVCpu);
7785	#endif
7786
7787	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7788	return rc;
7789	}
7790
7791
7792	/**
7793	* Loads the state shared between the host and guest into the VMCS.
7794	*
7795	* @param pVM Pointer to the VM.
7796	* @param pVCpu Pointer to the VMCPU.
7797	* @param pCtx Pointer to the guest-CPU context.
7798	*
7799	* @remarks No-long-jump zone!!!
7800	*/
7801	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7802	{
7803	NOREF(pVM);
7804
7805	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7806	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7807
7808	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7809	{
7810	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7811	AssertRC(rc);
7812	}
7813
7814	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7815	{
7816	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7817	AssertRC(rc);
7818
7819	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7820	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7821	{
7822	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7823	AssertRC(rc);
7824	}
7825	}
7826
7827	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7828	{
7829	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7830	if ( HMVMX_IS_64BIT_HOST_MODE()
7831	&& pVM->hm.s.fAllow64BitGuests)
7832	{
7833	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7834	}
7835	#endif
7836	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7837	}
7838
7839	AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7840	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7841	}
7842
7843
7844	/**
7845	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7846	*
7847	* @param pVM Pointer to the VM.
7848	* @param pVCpu Pointer to the VMCPU.
7849	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7850	* out-of-sync. Make sure to update the required fields
7851	* before using them.
7852	*/
7853	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7854	{
7855	HMVMX_ASSERT_PREEMPT_SAFE();
7856
7857	Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7858	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7859	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7860	#endif
7861
7862	if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7863	{
7864	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7865	AssertRC(rc);
7866	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7867	}
7868	else if (VMCPU_HMCF_VALUE(pVCpu))
7869	{
7870	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7871	AssertRC(rc);
7872	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7873	}
7874
7875	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7876	AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7877	\|\| VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7878	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7879
7880	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7881	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7882	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7883	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7884	#endif
7885	}
7886
7887
7888	/**
7889	* Does the preparations before executing guest code in VT-x.
7890	*
7891	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7892	* recompiler. We must be cautious what we do here regarding committing
7893	* guest-state information into the VMCS assuming we assuredly execute the
7894	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7895	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7896	* so that the recompiler can (and should) use them when it resumes guest
7897	* execution. Otherwise such operations must be done when we can no longer
7898	* exit to ring-3.
7899	*
7900	* @returns Strict VBox status code.
7901	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7902	* have been disabled.
7903	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7904	* double-fault into the guest.
7905	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7906	*
7907	* @param pVM Pointer to the VM.
7908	* @param pVCpu Pointer to the VMCPU.
7909	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7910	* out-of-sync. Make sure to update the required fields
7911	* before using them.
7912	* @param pVmxTransient Pointer to the VMX transient structure.
7913	*/
7914	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7915	{
7916	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7917
7918	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7919	PGMRZDynMapFlushAutoSet(pVCpu);
7920	#endif
7921
7922	/* Check force flag actions that might require us to go back to ring-3. */
7923	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7924	if (rc != VINF_SUCCESS)
7925	return rc;
7926
7927	#ifndef IEM_VERIFICATION_MODE_FULL
7928	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7929	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7930	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7931	{
7932	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7933	RTGCPHYS GCPhysApicBase;
7934	GCPhysApicBase = pMixedCtx->msrApicBase;
7935	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7936
7937	/* Unalias any existing mapping. */
7938	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7939	AssertRCReturn(rc, rc);
7940
7941	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7942	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7943	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7944	AssertRCReturn(rc, rc);
7945
7946	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7947	}
7948	#endif /* !IEM_VERIFICATION_MODE_FULL */
7949
7950	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7951	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7952
7953	/*
7954	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7955	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7956	*/
7957	if (TRPMHasTrap(pVCpu))
7958	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7959	else if (!pVCpu->hm.s.Event.fPending)
7960	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7961
7962	/*
7963	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7964	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7965	*/
7966	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7967	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7968	{
7969	Assert(rc == VINF_EM_RESET);
7970	return rc;
7971	}
7972
7973	/*
7974	* No longjmps to ring-3 from this point on!!!
7975	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7976	* This also disables flushing of the R0-logger instance (if any).
7977	*/
7978	VMMRZCallRing3Disable(pVCpu);
7979
7980	/*
7981	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7982	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7983	*
7984	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7985	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7986	*
7987	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7988	* executing guest code.
7989	*/
7990	pVmxTransient->uEflags = ASMIntDisableFlags();
7991	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7992	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7993	{
7994	hmR0VmxClearEventVmcs(pVCpu);
7995	ASMSetFlags(pVmxTransient->uEflags);
7996	VMMRZCallRing3Enable(pVCpu);
7997	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7998	return VINF_EM_RAW_TO_R3;
7999	}
8000	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8001	{
8002	hmR0VmxClearEventVmcs(pVCpu);
8003	ASMSetFlags(pVmxTransient->uEflags);
8004	VMMRZCallRing3Enable(pVCpu);
8005	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8006	return VINF_EM_RAW_INTERRUPT;
8007	}
8008
8009	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8010	pVCpu->hm.s.Event.fPending = false;
8011
8012	return VINF_SUCCESS;
8013	}
8014
8015
8016	/**
8017	* Prepares to run guest code in VT-x and we've committed to doing so. This
8018	* means there is no backing out to ring-3 or anywhere else at this
8019	* point.
8020	*
8021	* @param pVM Pointer to the VM.
8022	* @param pVCpu Pointer to the VMCPU.
8023	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8024	* out-of-sync. Make sure to update the required fields
8025	* before using them.
8026	* @param pVmxTransient Pointer to the VMX transient structure.
8027	*
8028	* @remarks Called with preemption disabled.
8029	* @remarks No-long-jump zone!!!
8030	*/
8031	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8032	{
8033	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8034	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8035	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8036
8037	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8038	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8039
8040	/*
8041	* If we are injecting events to a real-on-v86 mode guest, we may have to update
8042	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8043	* Reload only the necessary state, the assertion will catch if other parts of the code
8044	* change.
8045	*/
8046	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8047	{
8048	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8049	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8050	}
8051
8052	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8053	if (!CPUMIsGuestFPUStateActive(pVCpu))
8054	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8055	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8056	#endif
8057
8058	if ( pVCpu->hm.s.fUseGuestFpu
8059	&& !CPUMIsGuestFPUStateActive(pVCpu))
8060	{
8061	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8062	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
8063	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8064	}
8065
8066	/*
8067	* The host MSR values the very first time around won't be updated, so we need to
8068	* fill those values in. Subsequently, it's updated as part of the host state.
8069	*/
8070	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8071	&& pVCpu->hm.s.vmx.cMsrs > 0)
8072	{
8073	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8074	}
8075
8076	/*
8077	* Load the host state bits as we may've been preempted (only happens when
8078	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8079	*/
8080	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8081	{
8082	/* This ASSUMES that pfnStartVM has been set up already. */
8083	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8084	AssertRC(rc);
8085	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8086	}
8087	Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8088
8089	/*
8090	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8091	*/
8092	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8093	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8094	AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
8095
8096	/* Store status of the shared guest-host state at the time of VM-entry. */
8097	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8098	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8099	{
8100	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8101	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8102	}
8103	else
8104	#endif
8105	{
8106	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8107	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8108	}
8109	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8110
8111	/*
8112	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8113	*/
8114	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8115	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8116
8117	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8118	RTCPUID idCurrentCpu = pCpu->idCpu;
8119	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8120	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8121	{
8122	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8123	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8124	}
8125
8126	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8127	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8128	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8129	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8130
8131	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8132
8133	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8134	to start executing. */
8135
8136	/*
8137	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8138	*/
8139	uint64_t uGuestTscAuxMsr;
8140	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8141	{
8142	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8143	{
8144	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &uGuestTscAuxMsr);
8145	AssertRC(rc2);
8146	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, uGuestTscAuxMsr, true /* fUpdateHostMsr */);
8147	}
8148	else
8149	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8150	}
8151	#ifdef VBOX_STRICT
8152	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8153	#endif
8154	}
8155
8156
8157	/**
8158	* Performs some essential restoration of state after running guest code in
8159	* VT-x.
8160	*
8161	* @param pVM Pointer to the VM.
8162	* @param pVCpu Pointer to the VMCPU.
8163	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8164	* out-of-sync. Make sure to update the required fields
8165	* before using them.
8166	* @param pVmxTransient Pointer to the VMX transient structure.
8167	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8168	*
8169	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8170	*
8171	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8172	* unconditionally when it is safe to do so.
8173	*/
8174	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8175	{
8176	NOREF(pVM);
8177
8178	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8179
8180	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8181	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8182	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8183	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8184	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8185
8186	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8187	{
8188	/** @todo Find a way to fix hardcoding a guestimate. */
8189	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8190	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8191	}
8192
8193	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8194	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8195	Assert(!(ASMGetFlags() & X86_EFL_IF));
8196	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8197
8198	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8199	if (CPUMIsGuestFPUStateActive(pVCpu))
8200	{
8201	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8202	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8203	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8204	}
8205	#endif
8206
8207	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8208	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8209	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8210	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8211
8212	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8213	uint32_t uExitReason;
8214	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8215	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8216	AssertRC(rc);
8217	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8218	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8219
8220	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8221	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8222	{
8223	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8224	pVmxTransient->fVMEntryFailed));
8225	return;
8226	}
8227
8228	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8229	{
8230	/* Update the guest interruptibility-state from the VMCS. */
8231	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8232	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8233	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8234	AssertRC(rc);
8235	#endif
8236	/*
8237	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8238	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8239	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8240	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8241	*/
8242	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8243	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8244	{
8245	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8246	AssertRC(rc);
8247	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8248	}
8249	}
8250	}
8251
8252
8253
8254	/**
8255	* Runs the guest code using VT-x the normal way.
8256	*
8257	* @returns VBox status code.
8258	* @param pVM Pointer to the VM.
8259	* @param pVCpu Pointer to the VMCPU.
8260	* @param pCtx Pointer to the guest-CPU context.
8261	*
8262	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8263	*/
8264	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8265	{
8266	VMXTRANSIENT VmxTransient;
8267	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8268	int rc = VERR_INTERNAL_ERROR_5;
8269	uint32_t cLoops = 0;
8270
8271	for (;; cLoops++)
8272	{
8273	Assert(!HMR0SuspendPending());
8274	HMVMX_ASSERT_CPU_SAFE();
8275
8276	/* Preparatory work for running guest code, this may force us to return
8277	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8278	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8279	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8280	if (rc != VINF_SUCCESS)
8281	break;
8282
8283	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8284	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8285	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8286
8287	/* Restore any residual host-state and save any bits shared between host
8288	and guest into the guest-CPU state. Re-enables interrupts! */
8289	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8290
8291	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8292	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8293	{
8294	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8295	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8296	return rc;
8297	}
8298
8299	/* Handle the VM-exit. */
8300	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8301	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8302	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8303	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8304	HMVMX_START_EXIT_DISPATCH_PROF();
8305	#ifdef HMVMX_USE_FUNCTION_TABLE
8306	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8307	#else
8308	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8309	#endif
8310	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8311	if (rc != VINF_SUCCESS)
8312	break;
8313	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8314	{
8315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8316	rc = VINF_EM_RAW_INTERRUPT;
8317	break;
8318	}
8319	}
8320
8321	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8322	return rc;
8323	}
8324
8325
8326	/**
8327	* Single steps guest code using VT-x.
8328	*
8329	* @returns VBox status code.
8330	* @param pVM Pointer to the VM.
8331	* @param pVCpu Pointer to the VMCPU.
8332	* @param pCtx Pointer to the guest-CPU context.
8333	*
8334	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8335	*/
8336	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8337	{
8338	VMXTRANSIENT VmxTransient;
8339	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8340	int rc = VERR_INTERNAL_ERROR_5;
8341	uint32_t cLoops = 0;
8342	uint16_t uCsStart = pCtx->cs.Sel;
8343	uint64_t uRipStart = pCtx->rip;
8344
8345	for (;; cLoops++)
8346	{
8347	Assert(!HMR0SuspendPending());
8348	HMVMX_ASSERT_CPU_SAFE();
8349
8350	/* Preparatory work for running guest code, this may force us to return
8351	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8352	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8353	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8354	if (rc != VINF_SUCCESS)
8355	break;
8356
8357	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8358	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8359	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8360
8361	/* Restore any residual host-state and save any bits shared between host
8362	and guest into the guest-CPU state. Re-enables interrupts! */
8363	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8364
8365	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8366	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8367	{
8368	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8369	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8370	return rc;
8371	}
8372
8373	/* Handle the VM-exit. */
8374	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8375	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8376	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8377	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8378	HMVMX_START_EXIT_DISPATCH_PROF();
8379	#ifdef HMVMX_USE_FUNCTION_TABLE
8380	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8381	#else
8382	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8383	#endif
8384	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8385	if (rc != VINF_SUCCESS)
8386	break;
8387	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8388	{
8389	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8390	rc = VINF_EM_RAW_INTERRUPT;
8391	break;
8392	}
8393
8394	/*
8395	* Did the RIP change, if so, consider it a single step.
8396	* Otherwise, make sure one of the TFs gets set.
8397	*/
8398	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8399	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8400	AssertRCReturn(rc2, rc2);
8401	if ( pCtx->rip != uRipStart
8402	\|\| pCtx->cs.Sel != uCsStart)
8403	{
8404	rc = VINF_EM_DBG_STEPPED;
8405	break;
8406	}
8407	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8408	}
8409
8410	/*
8411	* Clear the X86_EFL_TF if necessary.
8412	*/
8413	if (pVCpu->hm.s.fClearTrapFlag)
8414	{
8415	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8416	AssertRCReturn(rc2, rc2);
8417	pVCpu->hm.s.fClearTrapFlag = false;
8418	pCtx->eflags.Bits.u1TF = 0;
8419	}
8420	/** @todo there seems to be issues with the resume flag when the monitor trap
8421	* flag is pending without being used. Seen early in bios init when
8422	* accessing APIC page in prot mode. */
8423
8424	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8425	return rc;
8426	}
8427
8428
8429	/**
8430	* Runs the guest code using VT-x.
8431	*
8432	* @returns VBox status code.
8433	* @param pVM Pointer to the VM.
8434	* @param pVCpu Pointer to the VMCPU.
8435	* @param pCtx Pointer to the guest-CPU context.
8436	*/
8437	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8438	{
8439	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8440	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8441	HMVMX_ASSERT_PREEMPT_SAFE();
8442
8443	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8444
8445	int rc;
8446	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8447	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8448	else
8449	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8450
8451	if (rc == VERR_EM_INTERPRETER)
8452	rc = VINF_EM_RAW_EMULATE_INSTR;
8453	else if (rc == VINF_EM_RESET)
8454	rc = VINF_EM_TRIPLE_FAULT;
8455
8456	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8457	if (RT_FAILURE(rc2))
8458	{
8459	pVCpu->hm.s.u32HMError = rc;
8460	rc = rc2;
8461	}
8462	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8463	return rc;
8464	}
8465
8466
8467	#ifndef HMVMX_USE_FUNCTION_TABLE
8468	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8469	{
8470	#ifdef DEBUG_ramshankar
8471	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8472	# define LDVMCS() do { VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8473	#endif
8474	int rc;
8475	switch (rcReason)
8476	{
8477	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8478	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8479	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8480	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8481	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8482	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8483	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8484	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8485	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8486	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8487	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8488	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8489	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8490	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8491	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8492	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8493	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8494	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8495	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8496	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8497	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8498	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8499	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8500	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8501	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8502	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8503	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8504	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8505	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8506	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8507	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8508	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8509	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8510
8511	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8512	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8513	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8514	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8515	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8516	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8517	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8518	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8519	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8520
8521	case VMX_EXIT_VMCALL:
8522	case VMX_EXIT_VMCLEAR:
8523	case VMX_EXIT_VMLAUNCH:
8524	case VMX_EXIT_VMPTRLD:
8525	case VMX_EXIT_VMPTRST:
8526	case VMX_EXIT_VMREAD:
8527	case VMX_EXIT_VMRESUME:
8528	case VMX_EXIT_VMWRITE:
8529	case VMX_EXIT_VMXOFF:
8530	case VMX_EXIT_VMXON:
8531	case VMX_EXIT_INVEPT:
8532	case VMX_EXIT_INVVPID:
8533	case VMX_EXIT_VMFUNC:
8534	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8535	break;
8536	default:
8537	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8538	break;
8539	}
8540	return rc;
8541	}
8542	#endif
8543
8544	#ifdef DEBUG
8545	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8546	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8547	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8548
8549	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8550	do \
8551	{ \
8552	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8553	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8554	} while (0)
8555
8556	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8557	do { \
8558	AssertPtr(pVCpu); \
8559	AssertPtr(pMixedCtx); \
8560	AssertPtr(pVmxTransient); \
8561	Assert(pVmxTransient->fVMEntryFailed == false); \
8562	Assert(ASMIntAreEnabled()); \
8563	HMVMX_ASSERT_PREEMPT_SAFE(); \
8564	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8565	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8566	HMVMX_ASSERT_PREEMPT_SAFE(); \
8567	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8568	HMVMX_ASSERT_PREEMPT_CPUID(); \
8569	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8570	} while (0)
8571
8572	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8573	do { \
8574	Log4Func(("\n")); \
8575	} while (0)
8576	#else /* Release builds */
8577	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8578	do { \
8579	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8580	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8581	} while (0)
8582	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8583	#endif
8584
8585
8586	/**
8587	* Advances the guest RIP after reading it from the VMCS.
8588	*
8589	* @returns VBox status code.
8590	* @param pVCpu Pointer to the VMCPU.
8591	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8592	* out-of-sync. Make sure to update the required fields
8593	* before using them.
8594	* @param pVmxTransient Pointer to the VMX transient structure.
8595	*
8596	* @remarks No-long-jump zone!!!
8597	*/
8598	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8599	{
8600	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8601	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8602	AssertRCReturn(rc, rc);
8603
8604	pMixedCtx->rip += pVmxTransient->cbInstr;
8605	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8606	return rc;
8607	}
8608
8609
8610	/**
8611	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8612	* and update error record fields accordingly.
8613	*
8614	* @return VMX_IGS_* return codes.
8615	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8616	* wrong with the guest state.
8617	*
8618	* @param pVM Pointer to the VM.
8619	* @param pVCpu Pointer to the VMCPU.
8620	* @param pCtx Pointer to the guest-CPU state.
8621	*/
8622	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8623	{
8624	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8625	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8626	uError = (err); \
8627	break; \
8628	} else do { } while (0)
8629	/* Duplicate of IEM_IS_CANONICAL(). */
8630	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8631
8632	int rc;
8633	uint32_t uError = VMX_IGS_ERROR;
8634	uint32_t u32Val;
8635	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8636
8637	do
8638	{
8639	/*
8640	* CR0.
8641	*/
8642	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8643	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8644	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8645	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8646	if (fUnrestrictedGuest)
8647	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8648
8649	uint32_t u32GuestCR0;
8650	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8651	AssertRCBreak(rc);
8652	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8653	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8654	if ( !fUnrestrictedGuest
8655	&& (u32GuestCR0 & X86_CR0_PG)
8656	&& !(u32GuestCR0 & X86_CR0_PE))
8657	{
8658	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8659	}
8660
8661	/*
8662	* CR4.
8663	*/
8664	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8665	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8666
8667	uint32_t u32GuestCR4;
8668	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8669	AssertRCBreak(rc);
8670	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8671	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8672
8673	/*
8674	* IA32_DEBUGCTL MSR.
8675	*/
8676	uint64_t u64Val;
8677	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8678	AssertRCBreak(rc);
8679	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8680	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8681	{
8682	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8683	}
8684	uint64_t u64DebugCtlMsr = u64Val;
8685
8686	#ifdef VBOX_STRICT
8687	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8688	AssertRCBreak(rc);
8689	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8690	#endif
8691	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8692
8693	/*
8694	* RIP and RFLAGS.
8695	*/
8696	uint32_t u32Eflags;
8697	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8698	if (HMVMX_IS_64BIT_HOST_MODE())
8699	{
8700	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8701	AssertRCBreak(rc);
8702	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8703	if ( !fLongModeGuest
8704	\|\| !pCtx->cs.Attr.n.u1Long)
8705	{
8706	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8707	}
8708	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8709	* must be identical if the "IA32e mode guest" VM-entry control is 1
8710	* and CS.L is 1. No check applies if the CPU supports 64
8711	* linear-address bits. */
8712
8713	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8714	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8715	AssertRCBreak(rc);
8716	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8717	VMX_IGS_RFLAGS_RESERVED);
8718	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8719	u32Eflags = u64Val;
8720	}
8721	else
8722	#endif
8723	{
8724	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8725	AssertRCBreak(rc);
8726	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8727	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8728	}
8729
8730	if ( fLongModeGuest
8731	\|\| ( fUnrestrictedGuest
8732	&& !(u32GuestCR0 & X86_CR0_PE)))
8733	{
8734	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8735	}
8736
8737	uint32_t u32EntryInfo;
8738	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8739	AssertRCBreak(rc);
8740	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8741	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8742	{
8743	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8744	}
8745
8746	/*
8747	* 64-bit checks.
8748	*/
8749	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8750	if (HMVMX_IS_64BIT_HOST_MODE())
8751	{
8752	if ( fLongModeGuest
8753	&& !fUnrestrictedGuest)
8754	{
8755	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8756	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8757	}
8758
8759	if ( !fLongModeGuest
8760	&& (u32GuestCR4 & X86_CR4_PCIDE))
8761	{
8762	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8763	}
8764
8765	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8766	* 51:32 beyond the processor's physical-address width are 0. */
8767
8768	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8769	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8770	{
8771	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8772	}
8773
8774	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8775	AssertRCBreak(rc);
8776	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8777
8778	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8779	AssertRCBreak(rc);
8780	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8781	}
8782	#endif
8783
8784	/*
8785	* PERF_GLOBAL MSR.
8786	*/
8787	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8788	{
8789	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8790	AssertRCBreak(rc);
8791	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8792	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8793	}
8794
8795	/*
8796	* PAT MSR.
8797	*/
8798	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8799	{
8800	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8801	AssertRCBreak(rc);
8802	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8803	for (unsigned i = 0; i < 8; i++)
8804	{
8805	uint8_t u8Val = (u64Val & 0x7);
8806	if ( u8Val != 0 /* UC */
8807	\|\| u8Val != 1 /* WC */
8808	\|\| u8Val != 4 /* WT */
8809	\|\| u8Val != 5 /* WP */
8810	\|\| u8Val != 6 /* WB */
8811	\|\| u8Val != 7 /* UC- */)
8812	{
8813	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8814	}
8815	u64Val >>= 3;
8816	}
8817	}
8818
8819	/*
8820	* EFER MSR.
8821	*/
8822	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8823	{
8824	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8825	AssertRCBreak(rc);
8826	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8827	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8828	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8829	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8830	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8831	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8832	}
8833
8834	/*
8835	* Segment registers.
8836	*/
8837	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8838	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8839	if (!(u32Eflags & X86_EFL_VM))
8840	{
8841	/* CS */
8842	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8843	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8844	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8845	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8846	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8847	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8848	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8849	/* CS cannot be loaded with NULL in protected mode. */
8850	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8851	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8852	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8853	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8854	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8855	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8856	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8857	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8858	else
8859	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8860
8861	/* SS */
8862	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8863	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8864	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8865	if ( !(pCtx->cr0 & X86_CR0_PE)
8866	\|\| pCtx->cs.Attr.n.u4Type == 3)
8867	{
8868	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8869	}
8870	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8871	{
8872	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8873	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8874	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8875	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8876	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8877	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8878	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8879	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8880	}
8881
8882	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8883	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8884	{
8885	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8886	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8887	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8888	\|\| pCtx->ds.Attr.n.u4Type > 11
8889	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8890	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8891	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8892	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8893	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8894	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8895	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8896	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8897	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8898	}
8899	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8900	{
8901	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8902	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8903	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8904	\|\| pCtx->es.Attr.n.u4Type > 11
8905	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8906	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8907	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8908	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8909	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8910	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8911	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8912	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8913	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8914	}
8915	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8916	{
8917	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8918	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8919	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8920	\|\| pCtx->fs.Attr.n.u4Type > 11
8921	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8922	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8923	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8924	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8925	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8926	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8927	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8928	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8929	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8930	}
8931	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8932	{
8933	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8934	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8935	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8936	\|\| pCtx->gs.Attr.n.u4Type > 11
8937	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8938	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8939	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8940	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8941	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8942	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8943	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8944	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8945	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8946	}
8947	/* 64-bit capable CPUs. */
8948	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8949	if (HMVMX_IS_64BIT_HOST_MODE())
8950	{
8951	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8952	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8953	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8954	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8955	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8956	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8957	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8958	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8959	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8960	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8961	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8962	}
8963	#endif
8964	}
8965	else
8966	{
8967	/* V86 mode checks. */
8968	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8969	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8970	{
8971	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8972	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8973	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8974	}
8975	else
8976	{
8977	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8978	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8979	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8980	}
8981
8982	/* CS */
8983	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8984	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8985	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8986	/* SS */
8987	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8988	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8989	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8990	/* DS */
8991	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8992	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8993	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8994	/* ES */
8995	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8996	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8997	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8998	/* FS */
8999	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9000	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9001	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9002	/* GS */
9003	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9004	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9005	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9006	/* 64-bit capable CPUs. */
9007	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9008	if (HMVMX_IS_64BIT_HOST_MODE())
9009	{
9010	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9011	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9012	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9013	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9014	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9015	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9016	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9017	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9018	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9019	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9020	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9021	}
9022	#endif
9023	}
9024
9025	/*
9026	* TR.
9027	*/
9028	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9029	/* 64-bit capable CPUs. */
9030	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9031	if (HMVMX_IS_64BIT_HOST_MODE())
9032	{
9033	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9034	}
9035	#endif
9036	if (fLongModeGuest)
9037	{
9038	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9039	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9040	}
9041	else
9042	{
9043	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9044	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9045	VMX_IGS_TR_ATTR_TYPE_INVALID);
9046	}
9047	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9048	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9049	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9050	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9051	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9052	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9053	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9054	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9055
9056	/*
9057	* GDTR and IDTR.
9058	*/
9059	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9060	if (HMVMX_IS_64BIT_HOST_MODE())
9061	{
9062	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9063	AssertRCBreak(rc);
9064	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9065
9066	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9067	AssertRCBreak(rc);
9068	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9069	}
9070	#endif
9071
9072	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9073	AssertRCBreak(rc);
9074	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9075
9076	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9077	AssertRCBreak(rc);
9078	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9079
9080	/*
9081	* Guest Non-Register State.
9082	*/
9083	/* Activity State. */
9084	uint32_t u32ActivityState;
9085	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9086	AssertRCBreak(rc);
9087	HMVMX_CHECK_BREAK( !u32ActivityState
9088	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9089	VMX_IGS_ACTIVITY_STATE_INVALID);
9090	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9091	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9092	uint32_t u32IntrState;
9093	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9094	AssertRCBreak(rc);
9095	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9096	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9097	{
9098	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9099	}
9100
9101	/** @todo Activity state and injecting interrupts. Left as a todo since we
9102	* currently don't use activity states but ACTIVE. */
9103
9104	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9105	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9106
9107	/* Guest interruptibility-state. */
9108	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9109	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9110	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9111	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9112	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9113	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9114	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9115	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9116	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9117	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9118	{
9119	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9120	{
9121	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9122	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9123	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9124	}
9125	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9126	{
9127	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9128	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9129	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9130	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9131	}
9132	}
9133	/** @todo Assumes the processor is not in SMM. */
9134	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9135	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9136	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9137	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9138	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9139	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9140	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9141	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9142	{
9143	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9144	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9145	}
9146
9147	/* Pending debug exceptions. */
9148	if (HMVMX_IS_64BIT_HOST_MODE())
9149	{
9150	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9151	AssertRCBreak(rc);
9152	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9153	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9154	u32Val = u64Val; /* For pending debug exceptions checks below. */
9155	}
9156	else
9157	{
9158	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9159	AssertRCBreak(rc);
9160	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9161	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9162	}
9163
9164	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9165	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9166	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9167	{
9168	if ( (u32Eflags & X86_EFL_TF)
9169	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9170	{
9171	/* Bit 14 is PendingDebug.BS. */
9172	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9173	}
9174	if ( !(u32Eflags & X86_EFL_TF)
9175	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9176	{
9177	/* Bit 14 is PendingDebug.BS. */
9178	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9179	}
9180	}
9181
9182	/* VMCS link pointer. */
9183	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9184	AssertRCBreak(rc);
9185	if (u64Val != UINT64_C(0xffffffffffffffff))
9186	{
9187	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9188	/** @todo Bits beyond the processor's physical-address width MBZ. */
9189	/** @todo 32-bit located in memory referenced by value of this field (as a
9190	* physical address) must contain the processor's VMCS revision ID. */
9191	/** @todo SMM checks. */
9192	}
9193
9194	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
9195
9196	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9197	if (uError == VMX_IGS_ERROR)
9198	uError = VMX_IGS_REASON_NOT_FOUND;
9199	} while (0);
9200
9201	pVCpu->hm.s.u32HMError = uError;
9202	return uError;
9203
9204	#undef HMVMX_ERROR_BREAK
9205	#undef HMVMX_CHECK_BREAK
9206	#undef HMVMX_IS_CANONICAL
9207	}
9208
9209	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9210	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9211	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9212
9213	/** @name VM-exit handlers.
9214	* @{
9215	*/
9216
9217	/**
9218	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9219	*/
9220	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9221	{
9222	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9223	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9224	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9225	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9226	return VINF_SUCCESS;
9227	return VINF_EM_RAW_INTERRUPT;
9228	}
9229
9230
9231	/**
9232	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9233	*/
9234	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9235	{
9236	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9237	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9238
9239	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9240	AssertRCReturn(rc, rc);
9241
9242	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9243	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9244	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9245	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9246
9247	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9248	{
9249	/*
9250	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9251	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9252	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9253	*
9254	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9255	*/
9256	VMXDispatchHostNmi();
9257	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9258	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9259	return VINF_SUCCESS;
9260	}
9261
9262	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9263	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9264	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9265	{
9266	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9267	return VINF_SUCCESS;
9268	}
9269	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9270	{
9271	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9272	return rc;
9273	}
9274
9275	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9276	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9277	switch (uIntType)
9278	{
9279	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9280	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9281	/* no break */
9282	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9283	{
9284	switch (uVector)
9285	{
9286	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9287	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9288	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9289	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9290	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9291	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9292	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9293	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9294	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9295	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9296	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9297	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9298	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9299	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9300	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9301	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9302	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9303	#endif
9304	default:
9305	{
9306	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9307	AssertRCReturn(rc, rc);
9308
9309	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9310	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9311	{
9312	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9313	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9314	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9315
9316	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9317	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9318	AssertRCReturn(rc, rc);
9319	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9320	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9321	0 /* GCPtrFaultAddress */);
9322	AssertRCReturn(rc, rc);
9323	}
9324	else
9325	{
9326	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9327	pVCpu->hm.s.u32HMError = uVector;
9328	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9329	}
9330	break;
9331	}
9332	}
9333	break;
9334	}
9335
9336	default:
9337	{
9338	pVCpu->hm.s.u32HMError = uExitIntInfo;
9339	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9340	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9341	break;
9342	}
9343	}
9344	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9345	return rc;
9346	}
9347
9348
9349	/**
9350	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9351	*/
9352	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9353	{
9354	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9355
9356	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9357	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9358	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9359	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9360	AssertRCReturn(rc, rc);
9361
9362	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9363	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9364	return VINF_SUCCESS;
9365	}
9366
9367
9368	/**
9369	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9370	*/
9371	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9372	{
9373	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9374	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9375	HMVMX_RETURN_UNEXPECTED_EXIT();
9376	}
9377
9378
9379	/**
9380	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9381	*/
9382	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9383	{
9384	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9385	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9386	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9387	}
9388
9389
9390	/**
9391	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9392	*/
9393	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9394	{
9395	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9396	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9397	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9398	}
9399
9400
9401	/**
9402	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9403	*/
9404	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9405	{
9406	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9407	PVM pVM = pVCpu->CTX_SUFF(pVM);
9408	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9409	if (RT_LIKELY(rc == VINF_SUCCESS))
9410	{
9411	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9412	Assert(pVmxTransient->cbInstr == 2);
9413	}
9414	else
9415	{
9416	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9417	rc = VERR_EM_INTERPRETER;
9418	}
9419	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9420	return rc;
9421	}
9422
9423
9424	/**
9425	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9426	*/
9427	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9428	{
9429	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9430	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9431	AssertRCReturn(rc, rc);
9432
9433	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9434	return VINF_EM_RAW_EMULATE_INSTR;
9435
9436	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9437	HMVMX_RETURN_UNEXPECTED_EXIT();
9438	}
9439
9440
9441	/**
9442	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9443	*/
9444	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9445	{
9446	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9447	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9448	AssertRCReturn(rc, rc);
9449
9450	PVM pVM = pVCpu->CTX_SUFF(pVM);
9451	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9452	if (RT_LIKELY(rc == VINF_SUCCESS))
9453	{
9454	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9455	Assert(pVmxTransient->cbInstr == 2);
9456	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9457	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9458	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9459	}
9460	else
9461	{
9462	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9463	rc = VERR_EM_INTERPRETER;
9464	}
9465	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9466	return rc;
9467	}
9468
9469
9470	/**
9471	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9472	*/
9473	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9474	{
9475	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9476	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9477	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9478	AssertRCReturn(rc, rc);
9479
9480	PVM pVM = pVCpu->CTX_SUFF(pVM);
9481	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9482	if (RT_LIKELY(rc == VINF_SUCCESS))
9483	{
9484	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9485	Assert(pVmxTransient->cbInstr == 3);
9486	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9487	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9488	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9489	}
9490	else
9491	{
9492	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9493	rc = VERR_EM_INTERPRETER;
9494	}
9495	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9496	return rc;
9497	}
9498
9499
9500	/**
9501	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9502	*/
9503	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9504	{
9505	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9506	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9507	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9508	AssertRCReturn(rc, rc);
9509
9510	PVM pVM = pVCpu->CTX_SUFF(pVM);
9511	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9512	if (RT_LIKELY(rc == VINF_SUCCESS))
9513	{
9514	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9515	Assert(pVmxTransient->cbInstr == 2);
9516	}
9517	else
9518	{
9519	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9520	rc = VERR_EM_INTERPRETER;
9521	}
9522	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9523	return rc;
9524	}
9525
9526
9527	/**
9528	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9529	*/
9530	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9531	{
9532	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9533	PVM pVM = pVCpu->CTX_SUFF(pVM);
9534	Assert(!pVM->hm.s.fNestedPaging);
9535
9536	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9537	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9538	AssertRCReturn(rc, rc);
9539
9540	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9541	rc = VBOXSTRICTRC_VAL(rc2);
9542	if (RT_LIKELY(rc == VINF_SUCCESS))
9543	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9544	else
9545	{
9546	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9547	pVmxTransient->uExitQualification, rc));
9548	}
9549	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9550	return rc;
9551	}
9552
9553
9554	/**
9555	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9556	*/
9557	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9558	{
9559	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9560	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9561	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9562	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9563	AssertRCReturn(rc, rc);
9564
9565	PVM pVM = pVCpu->CTX_SUFF(pVM);
9566	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9567	if (RT_LIKELY(rc == VINF_SUCCESS))
9568	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9569	else
9570	{
9571	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9572	rc = VERR_EM_INTERPRETER;
9573	}
9574	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9575	return rc;
9576	}
9577
9578
9579	/**
9580	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9581	*/
9582	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9583	{
9584	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9585	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9586	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9587	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9588	AssertRCReturn(rc, rc);
9589
9590	PVM pVM = pVCpu->CTX_SUFF(pVM);
9591	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9592	rc = VBOXSTRICTRC_VAL(rc2);
9593	if (RT_LIKELY( rc == VINF_SUCCESS
9594	\|\| rc == VINF_EM_HALT))
9595	{
9596	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9597	AssertRCReturn(rc3, rc3);
9598
9599	if ( rc == VINF_EM_HALT
9600	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9601	{
9602	rc = VINF_SUCCESS;
9603	}
9604	}
9605	else
9606	{
9607	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9608	rc = VERR_EM_INTERPRETER;
9609	}
9610	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9611	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9612	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9613	return rc;
9614	}
9615
9616
9617	/**
9618	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9619	*/
9620	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9621	{
9622	/*
9623	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9624	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9625	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9626	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9627	*/
9628	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9629	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9630	HMVMX_RETURN_UNEXPECTED_EXIT();
9631	}
9632
9633
9634	/**
9635	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9636	*/
9637	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9638	{
9639	/*
9640	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9641	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9642	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9643	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9644	*/
9645	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9646	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9647	HMVMX_RETURN_UNEXPECTED_EXIT();
9648	}
9649
9650
9651	/**
9652	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9653	*/
9654	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9655	{
9656	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9657	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9658	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9659	HMVMX_RETURN_UNEXPECTED_EXIT();
9660	}
9661
9662
9663	/**
9664	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9665	*/
9666	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9667	{
9668	/*
9669	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9670	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9671	* See Intel spec. 25.3 "Other Causes of VM-exits".
9672	*/
9673	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9674	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9675	HMVMX_RETURN_UNEXPECTED_EXIT();
9676	}
9677
9678
9679	/**
9680	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9681	* VM-exit.
9682	*/
9683	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9684	{
9685	/*
9686	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9687	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9688	*
9689	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9690	* See Intel spec. "23.8 Restrictions on VMX operation".
9691	*/
9692	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9693	return VINF_SUCCESS;
9694	}
9695
9696
9697	/**
9698	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9699	* VM-exit.
9700	*/
9701	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9702	{
9703	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9704	return VINF_EM_RESET;
9705	}
9706
9707
9708	/**
9709	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9710	*/
9711	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9712	{
9713	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9714	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9715	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9716	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9717	AssertRCReturn(rc, rc);
9718
9719	pMixedCtx->rip++;
9720	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9721	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9722	rc = VINF_SUCCESS;
9723	else
9724	rc = VINF_EM_HALT;
9725
9726	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9727	return rc;
9728	}
9729
9730
9731	/**
9732	* VM-exit handler for instructions that result in a #UD exception delivered to
9733	* the guest.
9734	*/
9735	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9736	{
9737	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9738	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9739	return VINF_SUCCESS;
9740	}
9741
9742
9743	/**
9744	* VM-exit handler for expiry of the VMX preemption timer.
9745	*/
9746	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9747	{
9748	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9749
9750	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9751	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9752
9753	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9754	PVM pVM = pVCpu->CTX_SUFF(pVM);
9755	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9756	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9757	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9758	}
9759
9760
9761	/**
9762	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9763	*/
9764	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9765	{
9766	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9767
9768	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9769	/** @todo check if XSETBV is supported by the recompiler. */
9770	return VERR_EM_INTERPRETER;
9771	}
9772
9773
9774	/**
9775	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9776	*/
9777	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9778	{
9779	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9780
9781	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9782	/** @todo implement EMInterpretInvpcid() */
9783	return VERR_EM_INTERPRETER;
9784	}
9785
9786
9787	/**
9788	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9789	* Error VM-exit.
9790	*/
9791	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9792	{
9793	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9794	AssertRCReturn(rc, rc);
9795
9796	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9797	NOREF(uInvalidReason);
9798
9799	#ifdef VBOX_STRICT
9800	uint32_t uIntrState;
9801	HMVMXHCUINTREG uHCReg;
9802	uint64_t u64Val;
9803	uint32_t u32Val;
9804
9805	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9806	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9807	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9808	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9809	AssertRCReturn(rc, rc);
9810
9811	Log4(("uInvalidReason %u\n", uInvalidReason));
9812	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9813	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9814	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9815	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9816
9817	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9818	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9819	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9820	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9821	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9822	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9823	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9824	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9825	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9826	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9827	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9828	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9829	#else
9830	NOREF(pVmxTransient);
9831	#endif
9832
9833	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9834	return VERR_VMX_INVALID_GUEST_STATE;
9835	}
9836
9837
9838	/**
9839	* VM-exit handler for VM-entry failure due to an MSR-load
9840	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9841	*/
9842	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9843	{
9844	NOREF(pVmxTransient);
9845	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9846	HMVMX_RETURN_UNEXPECTED_EXIT();
9847	}
9848
9849
9850	/**
9851	* VM-exit handler for VM-entry failure due to a machine-check event
9852	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9853	*/
9854	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9855	{
9856	NOREF(pVmxTransient);
9857	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9858	HMVMX_RETURN_UNEXPECTED_EXIT();
9859	}
9860
9861
9862	/**
9863	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9864	* theory.
9865	*/
9866	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9867	{
9868	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9869	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9870	return VERR_VMX_UNDEFINED_EXIT_CODE;
9871	}
9872
9873
9874	/**
9875	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9876	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9877	* Conditional VM-exit.
9878	*/
9879	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9880	{
9881	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9882
9883	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9884	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9885	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9886	return VERR_EM_INTERPRETER;
9887	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9888	HMVMX_RETURN_UNEXPECTED_EXIT();
9889	}
9890
9891
9892	/**
9893	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9894	*/
9895	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9896	{
9897	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9898
9899	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9900	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9901	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9902	return VERR_EM_INTERPRETER;
9903	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9904	HMVMX_RETURN_UNEXPECTED_EXIT();
9905	}
9906
9907
9908	/**
9909	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9910	*/
9911	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9912	{
9913	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9914
9915	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9916	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9917	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9918	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9919	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9920	{
9921	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9922	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9923	}
9924	AssertRCReturn(rc, rc);
9925	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9926
9927	#ifdef VBOX_STRICT
9928	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
9929	&& hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9930	{
9931	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9932	HMVMX_RETURN_UNEXPECTED_EXIT();
9933	}
9934	#endif
9935
9936	PVM pVM = pVCpu->CTX_SUFF(pVM);
9937	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9938	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9939	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9940	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9941
9942	if (RT_LIKELY(rc == VINF_SUCCESS))
9943	{
9944	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9945	Assert(pVmxTransient->cbInstr == 2);
9946	}
9947	return rc;
9948	}
9949
9950
9951	/**
9952	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9953	*/
9954	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9955	{
9956	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9957	PVM pVM = pVCpu->CTX_SUFF(pVM);
9958	int rc = VINF_SUCCESS;
9959
9960	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9961	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9962	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9963	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9964	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9965	{
9966	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9967	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9968	}
9969	AssertRCReturn(rc, rc);
9970	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9971
9972	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9973	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9974	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9975
9976	if (RT_LIKELY(rc == VINF_SUCCESS))
9977	{
9978	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9979
9980	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9981	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9982	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9983	{
9984	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9985	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9986	EMInterpretWrmsr() changes it. */
9987	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9988	}
9989	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9990	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9991
9992	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
9993	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9994	{
9995	switch (pMixedCtx->ecx)
9996	{
9997	case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9998	case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9999	case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10000	case MSR_K8_FS_BASE: /* no break */
10001	case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10002	default:
10003	{
10004	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10005	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10006	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
10007	else if ( HMVMX_IS_64BIT_HOST_MODE()
10008	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10009	{
10010	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10011	}
10012	#endif
10013	break;
10014	}
10015	}
10016	}
10017	#ifdef VBOX_STRICT
10018	else
10019	{
10020	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10021	switch (pMixedCtx->ecx)
10022	{
10023	case MSR_IA32_SYSENTER_CS:
10024	case MSR_IA32_SYSENTER_EIP:
10025	case MSR_IA32_SYSENTER_ESP:
10026	case MSR_K8_FS_BASE:
10027	case MSR_K8_GS_BASE:
10028	{
10029	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10030	HMVMX_RETURN_UNEXPECTED_EXIT();
10031	}
10032
10033	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10034	default:
10035	{
10036	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10037	{
10038	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10039	pMixedCtx->ecx));
10040	HMVMX_RETURN_UNEXPECTED_EXIT();
10041	}
10042
10043	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
10044	if ( HMVMX_IS_64BIT_HOST_MODE()
10045	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10046	{
10047	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10048	HMVMX_RETURN_UNEXPECTED_EXIT();
10049	}
10050	#endif
10051	break;
10052	}
10053	}
10054	}
10055	#endif /* VBOX_STRICT */
10056	}
10057	return rc;
10058	}
10059
10060
10061	/**
10062	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10063	*/
10064	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10065	{
10066	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10067
10068	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10069	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10070	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10071	return VERR_EM_INTERPRETER;
10072	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10073	HMVMX_RETURN_UNEXPECTED_EXIT();
10074	}
10075
10076
10077	/**
10078	* VM-exit handler for when the TPR value is lowered below the specified
10079	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10080	*/
10081	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10082	{
10083	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10084	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10085
10086	/*
10087	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10088	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10089	* resume guest execution.
10090	*/
10091	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10092	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10093	return VINF_SUCCESS;
10094	}
10095
10096
10097	/**
10098	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10099	* VM-exit.
10100	*
10101	* @retval VINF_SUCCESS when guest execution can continue.
10102	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10103	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10104	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10105	* recompiler.
10106	*/
10107	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10108	{
10109	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10110	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10111	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10112	AssertRCReturn(rc, rc);
10113
10114	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10115	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10116	PVM pVM = pVCpu->CTX_SUFF(pVM);
10117	switch (uAccessType)
10118	{
10119	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10120	{
10121	#if 0
10122	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10123	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10124	#else
10125	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10126	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10127	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10128	#endif
10129	AssertRCReturn(rc, rc);
10130
10131	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10132	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10133	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10134	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
10135
10136	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10137	{
10138	case 0: /* CR0 */
10139	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10140	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10141	break;
10142	case 2: /* CR2 */
10143	/* Nothing to do here, CR2 it's not part of the VMCS. */
10144	break;
10145	case 3: /* CR3 */
10146	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10147	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10148	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10149	break;
10150	case 4: /* CR4 */
10151	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10152	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10153	break;
10154	case 8: /* CR8 */
10155	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10156	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10157	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10158	break;
10159	default:
10160	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10161	break;
10162	}
10163
10164	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10165	break;
10166	}
10167
10168	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10169	{
10170	/* EMInterpretCRxRead() requires EFER MSR, CS. */
10171	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10172	AssertRCReturn(rc, rc);
10173	Assert( !pVM->hm.s.fNestedPaging
10174	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10175	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10176
10177	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10178	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10179	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10180
10181	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10182	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10183	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10184	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10185	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10186	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10187	break;
10188	}
10189
10190	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10191	{
10192	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10193	AssertRCReturn(rc, rc);
10194	rc = EMInterpretCLTS(pVM, pVCpu);
10195	AssertRCReturn(rc, rc);
10196	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10197	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10198	Log4(("CRX CLTS write rc=%d\n", rc));
10199	break;
10200	}
10201
10202	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10203	{
10204	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10205	AssertRCReturn(rc, rc);
10206	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10207	if (RT_LIKELY(rc == VINF_SUCCESS))
10208	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10209	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10210	Log4(("CRX LMSW write rc=%d\n", rc));
10211	break;
10212	}
10213
10214	default:
10215	{
10216	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10217	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10218	}
10219	}
10220
10221	/* Validate possible error codes. */
10222	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10223	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10224	if (RT_SUCCESS(rc))
10225	{
10226	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10227	AssertRCReturn(rc2, rc2);
10228	}
10229
10230	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10231	return rc;
10232	}
10233
10234
10235	/**
10236	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10237	* VM-exit.
10238	*/
10239	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10240	{
10241	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10242	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10243
10244	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10245	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10246	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10247	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10248	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10249	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10250	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10251	AssertRCReturn(rc2, rc2);
10252
10253	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10254	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10255	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10256	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10257	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10258	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10259	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10260
10261	/* I/O operation lookup arrays. */
10262	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10263	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10264
10265	VBOXSTRICTRC rcStrict;
10266	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10267	const uint32_t cbInstr = pVmxTransient->cbInstr;
10268	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10269	PVM pVM = pVCpu->CTX_SUFF(pVM);
10270	if (fIOString)
10271	{
10272	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10273	/*
10274	* INS/OUTS - I/O String instruction.
10275	*
10276	* Use instruction-information if available, otherwise fall back on
10277	* interpreting the instruction.
10278	*/
10279	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10280	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10281	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10282	{
10283	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10284	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10285	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10286	AssertRCReturn(rc2, rc2);
10287	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10288	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10289	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10290	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10291	if (fIOWrite)
10292	{
10293	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10294	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10295	}
10296	else
10297	{
10298	/*
10299	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10300	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10301	* See Intel Instruction spec. for "INS".
10302	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10303	*/
10304	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10305	}
10306	}
10307	else
10308	{
10309	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10310	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10311	AssertRCReturn(rc2, rc2);
10312	rcStrict = IEMExecOne(pVCpu);
10313	}
10314	/** @todo IEM needs to be setting these flags somehow. */
10315	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10316	fUpdateRipAlready = true;
10317	#else
10318	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10319	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10320	if (RT_SUCCESS(rcStrict))
10321	{
10322	if (fIOWrite)
10323	{
10324	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10325	(DISCPUMODE)pDis->uAddrMode, cbValue);
10326	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10327	}
10328	else
10329	{
10330	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10331	(DISCPUMODE)pDis->uAddrMode, cbValue);
10332	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10333	}
10334	}
10335	else
10336	{
10337	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10338	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10339	}
10340	#endif
10341	}
10342	else
10343	{
10344	/*
10345	* IN/OUT - I/O instruction.
10346	*/
10347	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10348	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10349	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10350	if (fIOWrite)
10351	{
10352	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10353	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10354	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10355	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10356	}
10357	else
10358	{
10359	uint32_t u32Result = 0;
10360	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10361	if (IOM_SUCCESS(rcStrict))
10362	{
10363	/* Save result of I/O IN instr. in AL/AX/EAX. */
10364	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10365	}
10366	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10367	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10368	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10369	}
10370	}
10371
10372	if (IOM_SUCCESS(rcStrict))
10373	{
10374	if (!fUpdateRipAlready)
10375	{
10376	pMixedCtx->rip += cbInstr;
10377	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10378	}
10379
10380	/*
10381	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10382	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10383	*/
10384	if (fIOString)
10385	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10386
10387	/*
10388	* If any I/O breakpoints are armed, we need to check if one triggered
10389	* and take appropriate action.
10390	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10391	*/
10392	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10393	AssertRCReturn(rc2, rc2);
10394
10395	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10396	* execution engines about whether hyper BPs and such are pending. */
10397	uint32_t const uDr7 = pMixedCtx->dr[7];
10398	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10399	&& X86_DR7_ANY_RW_IO(uDr7)
10400	&& (pMixedCtx->cr4 & X86_CR4_DE))
10401	\|\| DBGFBpIsHwIoArmed(pVM)))
10402	{
10403	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10404
10405	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10406	VMMRZCallRing3Disable(pVCpu);
10407	HM_DISABLE_PREEMPT_IF_NEEDED();
10408
10409	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10410
10411	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10412	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10413	{
10414	/* Raise #DB. */
10415	if (fIsGuestDbgActive)
10416	ASMSetDR6(pMixedCtx->dr[6]);
10417	if (pMixedCtx->dr[7] != uDr7)
10418	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10419
10420	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10421	}
10422	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10423	else if ( rcStrict2 != VINF_SUCCESS
10424	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10425	rcStrict = rcStrict2;
10426
10427	HM_RESTORE_PREEMPT_IF_NEEDED();
10428	VMMRZCallRing3Enable(pVCpu);
10429	}
10430	}
10431
10432	#ifdef DEBUG
10433	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10434	Assert(!fIOWrite);
10435	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10436	Assert(fIOWrite);
10437	else
10438	{
10439	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10440	* statuses, that the VMM device and some others may return. See
10441	* IOM_SUCCESS() for guidance. */
10442	AssertMsg( RT_FAILURE(rcStrict)
10443	\|\| rcStrict == VINF_SUCCESS
10444	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10445	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10446	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10447	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10448	}
10449	#endif
10450
10451	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10452	return VBOXSTRICTRC_TODO(rcStrict);
10453	}
10454
10455
10456	/**
10457	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10458	* VM-exit.
10459	*/
10460	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10461	{
10462	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10463
10464	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10465	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10466	AssertRCReturn(rc, rc);
10467	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10468	{
10469	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10470	AssertRCReturn(rc, rc);
10471	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10472	{
10473	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10474
10475	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10476	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10477	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10478	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10479	{
10480	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10481	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10482
10483	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10484	Assert(!pVCpu->hm.s.Event.fPending);
10485	pVCpu->hm.s.Event.fPending = true;
10486	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10487	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10488	AssertRCReturn(rc, rc);
10489	if (fErrorCodeValid)
10490	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10491	else
10492	pVCpu->hm.s.Event.u32ErrCode = 0;
10493	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10494	&& uVector == X86_XCPT_PF)
10495	{
10496	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10497	}
10498
10499	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10500	}
10501	}
10502	}
10503
10504	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10505	* emulation. */
10506	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10507	return VERR_EM_INTERPRETER;
10508	}
10509
10510
10511	/**
10512	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10513	*/
10514	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10515	{
10516	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10517	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10518	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10519	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10520	AssertRCReturn(rc, rc);
10521	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10522	return VINF_EM_DBG_STEPPED;
10523	}
10524
10525
10526	/**
10527	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10528	*/
10529	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10530	{
10531	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10532
10533	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10534	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10535	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10536	return VINF_SUCCESS;
10537	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10538	return rc;
10539
10540	#if 0
10541	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10542	* just sync the whole thing. */
10543	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10544	#else
10545	/* Aggressive state sync. for now. */
10546	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10547	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10548	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10549	#endif
10550	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10551	AssertRCReturn(rc, rc);
10552
10553	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10554	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10555	switch (uAccessType)
10556	{
10557	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10558	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10559	{
10560	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10561	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10562	{
10563	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10564	}
10565
10566	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10567	GCPhys &= PAGE_BASE_GC_MASK;
10568	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10569	PVM pVM = pVCpu->CTX_SUFF(pVM);
10570	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10571	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10572
10573	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10574	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10575	CPUMCTX2CORE(pMixedCtx), GCPhys);
10576	rc = VBOXSTRICTRC_VAL(rc2);
10577	Log4(("ApicAccess rc=%d\n", rc));
10578	if ( rc == VINF_SUCCESS
10579	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10580	\|\| rc == VERR_PAGE_NOT_PRESENT)
10581	{
10582	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10583	\| HM_CHANGED_GUEST_RSP
10584	\| HM_CHANGED_GUEST_RFLAGS
10585	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10586	rc = VINF_SUCCESS;
10587	}
10588	break;
10589	}
10590
10591	default:
10592	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10593	rc = VINF_EM_RAW_EMULATE_INSTR;
10594	break;
10595	}
10596
10597	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10598	return rc;
10599	}
10600
10601
10602	/**
10603	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10604	* VM-exit.
10605	*/
10606	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10607	{
10608	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10609
10610	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10611	if (pVmxTransient->fWasGuestDebugStateActive)
10612	{
10613	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10614	HMVMX_RETURN_UNEXPECTED_EXIT();
10615	}
10616
10617	int rc = VERR_INTERNAL_ERROR_5;
10618	if ( !DBGFIsStepping(pVCpu)
10619	&& !pVCpu->hm.s.fSingleInstruction
10620	&& !pVmxTransient->fWasHyperDebugStateActive)
10621	{
10622	/* Don't intercept MOV DRx and #DB any more. */
10623	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10624	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10625	AssertRCReturn(rc, rc);
10626
10627	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10628	{
10629	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10630	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10631	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10632	AssertRCReturn(rc, rc);
10633	#endif
10634	}
10635
10636	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10637	VMMRZCallRing3Disable(pVCpu);
10638	HM_DISABLE_PREEMPT_IF_NEEDED();
10639
10640	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10641	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10642	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10643
10644	HM_RESTORE_PREEMPT_IF_NEEDED();
10645	VMMRZCallRing3Enable(pVCpu);
10646
10647	#ifdef VBOX_WITH_STATISTICS
10648	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10649	AssertRCReturn(rc, rc);
10650	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10651	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10652	else
10653	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10654	#endif
10655	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10656	return VINF_SUCCESS;
10657	}
10658
10659	/*
10660	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10661	* Update the segment registers and DR7 from the CPU.
10662	*/
10663	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10664	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10665	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10666	AssertRCReturn(rc, rc);
10667	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10668
10669	PVM pVM = pVCpu->CTX_SUFF(pVM);
10670	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10671	{
10672	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10673	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10674	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10675	if (RT_SUCCESS(rc))
10676	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10677	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10678	}
10679	else
10680	{
10681	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10682	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10683	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10684	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10685	}
10686
10687	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10688	if (RT_SUCCESS(rc))
10689	{
10690	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10691	AssertRCReturn(rc2, rc2);
10692	}
10693	return rc;
10694	}
10695
10696
10697	/**
10698	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10699	* Conditional VM-exit.
10700	*/
10701	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10702	{
10703	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10704	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10705
10706	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10707	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10708	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10709	return VINF_SUCCESS;
10710	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10711	return rc;
10712
10713	RTGCPHYS GCPhys = 0;
10714	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10715
10716	#if 0
10717	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10718	#else
10719	/* Aggressive state sync. for now. */
10720	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10721	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10722	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10723	#endif
10724	AssertRCReturn(rc, rc);
10725
10726	/*
10727	* If we succeed, resume guest execution.
10728	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10729	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10730	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10731	* weird case. See @bugref{6043}.
10732	*/
10733	PVM pVM = pVCpu->CTX_SUFF(pVM);
10734	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10735	rc = VBOXSTRICTRC_VAL(rc2);
10736	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10737	if ( rc == VINF_SUCCESS
10738	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10739	\|\| rc == VERR_PAGE_NOT_PRESENT)
10740	{
10741	/* Successfully handled MMIO operation. */
10742	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10743	\| HM_CHANGED_GUEST_RSP
10744	\| HM_CHANGED_GUEST_RFLAGS
10745	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10746	rc = VINF_SUCCESS;
10747	}
10748	return rc;
10749	}
10750
10751
10752	/**
10753	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10754	* VM-exit.
10755	*/
10756	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10757	{
10758	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10759	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10760
10761	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10762	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10763	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10764	return VINF_SUCCESS;
10765	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10766	return rc;
10767
10768	RTGCPHYS GCPhys = 0;
10769	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10770	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10771	#if 0
10772	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10773	#else
10774	/* Aggressive state sync. for now. */
10775	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10776	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10777	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10778	#endif
10779	AssertRCReturn(rc, rc);
10780
10781	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10782	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10783
10784	RTGCUINT uErrorCode = 0;
10785	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10786	uErrorCode \|= X86_TRAP_PF_ID;
10787	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10788	uErrorCode \|= X86_TRAP_PF_RW;
10789	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10790	uErrorCode \|= X86_TRAP_PF_P;
10791
10792	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10793
10794	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10795	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10796
10797	/* Handle the pagefault trap for the nested shadow table. */
10798	PVM pVM = pVCpu->CTX_SUFF(pVM);
10799	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10800	TRPMResetTrap(pVCpu);
10801
10802	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10803	if ( rc == VINF_SUCCESS
10804	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10805	\|\| rc == VERR_PAGE_NOT_PRESENT)
10806	{
10807	/* Successfully synced our nested page tables. */
10808	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10809	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10810	\| HM_CHANGED_GUEST_RSP
10811	\| HM_CHANGED_GUEST_RFLAGS);
10812	return VINF_SUCCESS;
10813	}
10814
10815	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10816	return rc;
10817	}
10818
10819	/** @} */
10820
10821	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10822	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10823	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10824
10825	/** @name VM-exit exception handlers.
10826	* @{
10827	*/
10828
10829	/**
10830	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10831	*/
10832	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10833	{
10834	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10835	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10836
10837	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10838	AssertRCReturn(rc, rc);
10839
10840	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10841	{
10842	/* Old-style FPU error reporting needs some extra work. */
10843	/** @todo don't fall back to the recompiler, but do it manually. */
10844	return VERR_EM_INTERPRETER;
10845	}
10846
10847	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10848	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10849	return rc;
10850	}
10851
10852
10853	/**
10854	* VM-exit exception handler for #BP (Breakpoint exception).
10855	*/
10856	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10857	{
10858	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10859	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10860
10861	/** @todo Try optimize this by not saving the entire guest state unless
10862	* really needed. */
10863	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10864	AssertRCReturn(rc, rc);
10865
10866	PVM pVM = pVCpu->CTX_SUFF(pVM);
10867	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10868	if (rc == VINF_EM_RAW_GUEST_TRAP)
10869	{
10870	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10871	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10872	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10873	AssertRCReturn(rc, rc);
10874
10875	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10876	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10877	}
10878
10879	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10880	return rc;
10881	}
10882
10883
10884	/**
10885	* VM-exit exception handler for #DB (Debug exception).
10886	*/
10887	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10888	{
10889	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10890	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10891	Log6(("XcptDB\n"));
10892
10893	/*
10894	* Get the DR6-like values from the exit qualification and pass it to DBGF
10895	* for processing.
10896	*/
10897	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10898	AssertRCReturn(rc, rc);
10899
10900	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10901	uint64_t uDR6 = X86_DR6_INIT_VAL;
10902	uDR6 \|= ( pVmxTransient->uExitQualification
10903	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10904
10905	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10906	if (rc == VINF_EM_RAW_GUEST_TRAP)
10907	{
10908	/*
10909	* The exception was for the guest. Update DR6, DR7.GD and
10910	* IA32_DEBUGCTL.LBR before forwarding it.
10911	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10912	*/
10913	VMMRZCallRing3Disable(pVCpu);
10914	HM_DISABLE_PREEMPT_IF_NEEDED();
10915
10916	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10917	pMixedCtx->dr[6] \|= uDR6;
10918	if (CPUMIsGuestDebugStateActive(pVCpu))
10919	ASMSetDR6(pMixedCtx->dr[6]);
10920
10921	HM_RESTORE_PREEMPT_IF_NEEDED();
10922	VMMRZCallRing3Enable(pVCpu);
10923
10924	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10925	AssertRCReturn(rc, rc);
10926
10927	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10928	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10929
10930	/* Paranoia. */
10931	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10932	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10933
10934	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10935	AssertRCReturn(rc, rc);
10936
10937	/*
10938	* Raise #DB in the guest.
10939	*/
10940	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10941	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10942	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10943	AssertRCReturn(rc, rc);
10944	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10945	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10946	return VINF_SUCCESS;
10947	}
10948
10949	/*
10950	* Not a guest trap, must be a hypervisor related debug event then.
10951	* Update DR6 in case someone is interested in it.
10952	*/
10953	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10954	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10955	CPUMSetHyperDR6(pVCpu, uDR6);
10956
10957	return rc;
10958	}
10959
10960
10961	/**
10962	* VM-exit exception handler for #NM (Device-not-available exception: floating
10963	* point exception).
10964	*/
10965	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10966	{
10967	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10968
10969	/* We require CR0 and EFER. EFER is always up-to-date. */
10970	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10971	AssertRCReturn(rc, rc);
10972
10973	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10974	VMMRZCallRing3Disable(pVCpu);
10975	HM_DISABLE_PREEMPT_IF_NEEDED();
10976
10977	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10978	if (pVmxTransient->fWasGuestFPUStateActive)
10979	{
10980	rc = VINF_EM_RAW_GUEST_TRAP;
10981	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10982	}
10983	else
10984	{
10985	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10986	Assert(!pVmxTransient->fWasGuestFPUStateActive);
10987	#endif
10988	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10989	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10990	}
10991
10992	HM_RESTORE_PREEMPT_IF_NEEDED();
10993	VMMRZCallRing3Enable(pVCpu);
10994
10995	if (rc == VINF_SUCCESS)
10996	{
10997	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
10998	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10999	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11000	pVCpu->hm.s.fUseGuestFpu = true;
11001	}
11002	else
11003	{
11004	/* Forward #NM to the guest. */
11005	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11006	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11007	AssertRCReturn(rc, rc);
11008	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11009	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11010	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11011	}
11012
11013	return VINF_SUCCESS;
11014	}
11015
11016
11017	/**
11018	* VM-exit exception handler for #GP (General-protection exception).
11019	*
11020	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11021	*/
11022	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11023	{
11024	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11025	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11026
11027	int rc = VERR_INTERNAL_ERROR_5;
11028	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11029	{
11030	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11031	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11032	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11033	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11034	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11035	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11036	AssertRCReturn(rc, rc);
11037	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11038	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11039	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11040	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11041	return rc;
11042	#else
11043	/* We don't intercept #GP. */
11044	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11045	NOREF(pVmxTransient);
11046	return VERR_VMX_UNEXPECTED_EXCEPTION;
11047	#endif
11048	}
11049
11050	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11051	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11052
11053	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11054	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11055	AssertRCReturn(rc, rc);
11056
11057	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11058	uint32_t cbOp = 0;
11059	PVM pVM = pVCpu->CTX_SUFF(pVM);
11060	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11061	if (RT_SUCCESS(rc))
11062	{
11063	rc = VINF_SUCCESS;
11064	Assert(cbOp == pDis->cbInstr);
11065	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11066	switch (pDis->pCurInstr->uOpcode)
11067	{
11068	case OP_CLI:
11069	{
11070	pMixedCtx->eflags.Bits.u1IF = 0;
11071	pMixedCtx->rip += pDis->cbInstr;
11072	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11073	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11074	break;
11075	}
11076
11077	case OP_STI:
11078	{
11079	pMixedCtx->eflags.Bits.u1IF = 1;
11080	pMixedCtx->rip += pDis->cbInstr;
11081	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11082	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11083	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11084	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11085	break;
11086	}
11087
11088	case OP_HLT:
11089	{
11090	rc = VINF_EM_HALT;
11091	pMixedCtx->rip += pDis->cbInstr;
11092	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11093	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11094	break;
11095	}
11096
11097	case OP_POPF:
11098	{
11099	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11100	uint32_t cbParm = 0;
11101	uint32_t uMask = 0;
11102	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11103	{
11104	cbParm = 4;
11105	uMask = 0xffffffff;
11106	}
11107	else
11108	{
11109	cbParm = 2;
11110	uMask = 0xffff;
11111	}
11112
11113	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11114	RTGCPTR GCPtrStack = 0;
11115	X86EFLAGS Eflags;
11116	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11117	&GCPtrStack);
11118	if (RT_SUCCESS(rc))
11119	{
11120	Assert(sizeof(Eflags.u32) >= cbParm);
11121	Eflags.u32 = 0;
11122	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11123	}
11124	if (RT_FAILURE(rc))
11125	{
11126	rc = VERR_EM_INTERPRETER;
11127	break;
11128	}
11129	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11130	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11131	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11132	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11133	pMixedCtx->esp += cbParm;
11134	pMixedCtx->esp &= uMask;
11135	pMixedCtx->rip += pDis->cbInstr;
11136
11137	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11138	\| HM_CHANGED_GUEST_RSP
11139	\| HM_CHANGED_GUEST_RFLAGS);
11140	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11141	break;
11142	}
11143
11144	case OP_PUSHF:
11145	{
11146	uint32_t cbParm = 0;
11147	uint32_t uMask = 0;
11148	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11149	{
11150	cbParm = 4;
11151	uMask = 0xffffffff;
11152	}
11153	else
11154	{
11155	cbParm = 2;
11156	uMask = 0xffff;
11157	}
11158
11159	/* Get the stack pointer & push the contents of eflags onto the stack. */
11160	RTGCPTR GCPtrStack = 0;
11161	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11162	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11163	if (RT_FAILURE(rc))
11164	{
11165	rc = VERR_EM_INTERPRETER;
11166	break;
11167	}
11168	X86EFLAGS Eflags = pMixedCtx->eflags;
11169	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11170	Eflags.Bits.u1RF = 0;
11171	Eflags.Bits.u1VM = 0;
11172
11173	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11174	if (RT_FAILURE(rc))
11175	{
11176	rc = VERR_EM_INTERPRETER;
11177	break;
11178	}
11179	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11180	pMixedCtx->esp -= cbParm;
11181	pMixedCtx->esp &= uMask;
11182	pMixedCtx->rip += pDis->cbInstr;
11183	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
11184	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11185	break;
11186	}
11187
11188	case OP_IRET:
11189	{
11190	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11191	* instruction reference. */
11192	RTGCPTR GCPtrStack = 0;
11193	uint32_t uMask = 0xffff;
11194	uint16_t aIretFrame[3];
11195	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11196	{
11197	rc = VERR_EM_INTERPRETER;
11198	break;
11199	}
11200	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11201	&GCPtrStack);
11202	if (RT_SUCCESS(rc))
11203	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11204	if (RT_FAILURE(rc))
11205	{
11206	rc = VERR_EM_INTERPRETER;
11207	break;
11208	}
11209	pMixedCtx->eip = 0;
11210	pMixedCtx->ip = aIretFrame[0];
11211	pMixedCtx->cs.Sel = aIretFrame[1];
11212	pMixedCtx->cs.ValidSel = aIretFrame[1];
11213	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11214	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11215	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11216	pMixedCtx->sp += sizeof(aIretFrame);
11217	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11218	\| HM_CHANGED_GUEST_SEGMENT_REGS
11219	\| HM_CHANGED_GUEST_RSP
11220	\| HM_CHANGED_GUEST_RFLAGS);
11221	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11222	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11223	break;
11224	}
11225
11226	case OP_INT:
11227	{
11228	uint16_t uVector = pDis->Param1.uValue & 0xff;
11229	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11230	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11231	break;
11232	}
11233
11234	case OP_INTO:
11235	{
11236	if (pMixedCtx->eflags.Bits.u1OF)
11237	{
11238	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11239	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11240	}
11241	break;
11242	}
11243
11244	default:
11245	{
11246	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11247	EMCODETYPE_SUPERVISOR);
11248	rc = VBOXSTRICTRC_VAL(rc2);
11249	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11250	Log4(("#GP rc=%Rrc\n", rc));
11251	break;
11252	}
11253	}
11254	}
11255	else
11256	rc = VERR_EM_INTERPRETER;
11257
11258	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11259	("#GP Unexpected rc=%Rrc\n", rc));
11260	return rc;
11261	}
11262
11263
11264	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11265	/**
11266	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11267	* the exception reported in the VMX transient structure back into the VM.
11268	*
11269	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11270	* up-to-date.
11271	*/
11272	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11273	{
11274	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11275
11276	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11277	hmR0VmxCheckExitDueToEventDelivery(). */
11278	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11279	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11280	AssertRCReturn(rc, rc);
11281	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11282
11283	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11284	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11285	return VINF_SUCCESS;
11286	}
11287	#endif
11288
11289
11290	/**
11291	* VM-exit exception handler for #PF (Page-fault exception).
11292	*/
11293	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11294	{
11295	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11296	PVM pVM = pVCpu->CTX_SUFF(pVM);
11297	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11298	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11299	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11300	AssertRCReturn(rc, rc);
11301
11302	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11303	if (pVM->hm.s.fNestedPaging)
11304	{
11305	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11306	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11307	{
11308	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11309	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11310	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11311	}
11312	else
11313	{
11314	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11315	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11316	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11317	}
11318	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11319	return rc;
11320	}
11321	#else
11322	Assert(!pVM->hm.s.fNestedPaging);
11323	NOREF(pVM);
11324	#endif
11325
11326	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11327	AssertRCReturn(rc, rc);
11328
11329	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11330	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11331
11332	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11333	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11334	(RTGCPTR)pVmxTransient->uExitQualification);
11335
11336	Log4(("#PF: rc=%Rrc\n", rc));
11337	if (rc == VINF_SUCCESS)
11338	{
11339	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11340	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11341	* memory? We don't update the whole state here... */
11342	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11343	\| HM_CHANGED_GUEST_RSP
11344	\| HM_CHANGED_GUEST_RFLAGS
11345	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11346	TRPMResetTrap(pVCpu);
11347	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11348	return rc;
11349	}
11350	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11351	{
11352	if (!pVmxTransient->fVectoringPF)
11353	{
11354	/* It's a guest page fault and needs to be reflected to the guest. */
11355	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11356	TRPMResetTrap(pVCpu);
11357	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11358	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11359	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11360	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11361	}
11362	else
11363	{
11364	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11365	TRPMResetTrap(pVCpu);
11366	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11367	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11368	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11369	}
11370
11371	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11372	return VINF_SUCCESS;
11373	}
11374
11375	TRPMResetTrap(pVCpu);
11376	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11377	return rc;
11378	}
11379
11380	/** @} */
11381

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49670

Download in other formats: