HMVMXR0.cpp@ 49159

Last change on this file since 49159 was 49152, checked in by vboxsync, 11 years ago
HMR0VMX.cpp: Clear the TF bit in the single step loop, instead on later in the common exit-to-ring3 code.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 454.0 KB

Line
1	/* $Id: HMVMXR0.cpp 49152 2013-10-17 07:30:41Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	#define HMVMX_SAVE_FULL_GUEST_STATE
40	#define HMVMX_SYNC_FULL_GUEST_STATE
41	#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	#define HMVMX_ALWAYS_TRAP_PF
44	#define HMVMX_ALWAYS_SWAP_FPU_STATE
45	#endif
46
47
48	/*******************************************************************************
49	* Defined Constants And Macros *
50	*******************************************************************************/
51	#if defined(RT_ARCH_AMD64)
52	# define HMVMX_IS_64BIT_HOST_MODE() (true)
53	typedef RTHCUINTREG HMVMXHCUINTREG;
54	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
55	extern "C" uint32_t g_fVMXIs64bitHost;
56	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
57	typedef uint64_t HMVMXHCUINTREG;
58	#else
59	# define HMVMX_IS_64BIT_HOST_MODE() (false)
60	typedef RTHCUINTREG HMVMXHCUINTREG;
61	#endif
62
63	/** Use the function table. */
64	#define HMVMX_USE_FUNCTION_TABLE
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127	/** @} */
128
129	/** @name
130	* States of the VMCS.
131	*
132	* This does not reflect all possible VMCS states but currently only those
133	* needed for maintaining the VMCS consistently even when thread-context hooks
134	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
135	*/
136	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139	/** @} */
140
141	/**
142	* Exception bitmap mask for real-mode guests (real-on-v86).
143	*
144	* We need to intercept all exceptions manually (except #PF). #NM is also
145	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146	* even in real-mode if we have Nested Paging support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*******************************************************************************
197	* Structures and Typedefs *
198	*******************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG uEflags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
213	uint64_t u64LStarMsr;
214	/** The guest's TPR value used for TPR shadowing. */
215	uint8_t u8GuestTpr;
216	/** Alignment. */
217	uint8_t abAlignment0[7];
218
219	/** The basic VM-exit reason. */
220	uint16_t uExitReason;
221	/** Alignment. */
222	uint16_t u16Alignment0;
223	/** The VM-exit interruption error code. */
224	uint32_t uExitIntErrorCode;
225	/** The VM-exit exit qualification. */
226	uint64_t uExitQualification;
227
228	/** The VM-exit interruption-information field. */
229	uint32_t uExitIntInfo;
230	/** The VM-exit instruction-length field. */
231	uint32_t cbInstr;
232	/** The VM-exit instruction-information field. */
233	union
234	{
235	/** Plain unsigned int representation. */
236	uint32_t u;
237	/** INS and OUTS information. */
238	struct
239	{
240	uint32_t u6Reserved0 : 7;
241	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
242	uint32_t u3AddrSize : 3;
243	uint32_t u5Reserved1 : 5;
244	/** The segment register (X86_SREG_XXX). */
245	uint32_t iSegReg : 3;
246	uint32_t uReserved2 : 14;
247	} StrIo;
248	} ExitInstrInfo;
249	/** Whether the VM-entry failed or not. */
250	bool fVMEntryFailed;
251	/** Alignment. */
252	uint8_t abAlignment1[3];
253
254	/** The VM-entry interruption-information field. */
255	uint32_t uEntryIntInfo;
256	/** The VM-entry exception error code field. */
257	uint32_t uEntryXcptErrorCode;
258	/** The VM-entry instruction length field. */
259	uint32_t cbEntryInstr;
260
261	/** IDT-vectoring information field. */
262	uint32_t uIdtVectoringInfo;
263	/** IDT-vectoring error code. */
264	uint32_t uIdtVectoringErrorCode;
265
266	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
267	uint32_t fVmcsFieldsRead;
268
269	/** Whether the guest FPU was active at the time of VM-exit. */
270	bool fWasGuestFPUStateActive;
271	/** Whether the guest debug state was active at the time of VM-exit. */
272	bool fWasGuestDebugStateActive;
273	/** Whether the hyper debug state was active at the time of VM-exit. */
274	bool fWasHyperDebugStateActive;
275	/** Whether TSC-offsetting should be setup before VM-entry. */
276	bool fUpdateTscOffsettingAndPreemptTimer;
277	/** Whether the VM-exit was caused by a page-fault during delivery of a
278	* contributory exception or a page-fault. */
279	bool fVectoringPF;
280	} VMXTRANSIENT;
281	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286	/** Pointer to VMX transient state. */
287	typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290	/**
291	* MSR-bitmap read permissions.
292	*/
293	typedef enum VMXMSREXITREAD
294	{
295	/** Reading this MSR causes a VM-exit. */
296	VMXMSREXIT_INTERCEPT_READ = 0xb,
297	/** Reading this MSR does not cause a VM-exit. */
298	VMXMSREXIT_PASSTHRU_READ
299	} VMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311
312	/**
313	* VMX VM-exit handler.
314	*
315	* @returns VBox status code.
316	* @param pVCpu Pointer to the VMCPU.
317	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
318	* out-of-sync. Make sure to update the required
319	* fields before using them.
320	* @param pVmxTransient Pointer to the VMX-transient structure.
321	*/
322	#ifndef HMVMX_USE_FUNCTION_TABLE
323	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
324	#else
325	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
326	/** Pointer to VM-exit handler. */
327	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
328	#endif
329
330
331	/*******************************************************************************
332	* Internal Functions *
333	*******************************************************************************/
334	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
335	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
336	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
337	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
338	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
339	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
340	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
341	#endif
342	#ifndef HMVMX_USE_FUNCTION_TABLE
343	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
344	# define HMVMX_EXIT_DECL static int
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
347	#endif
348
349	/** @name VM-exit handlers.
350	* @{
351	*/
352	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
353	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
354	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
355	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
356	static FNVMXEXITHANDLER hmR0VmxExitSipi;
357	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
358	static FNVMXEXITHANDLER hmR0VmxExitSmi;
359	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
360	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
361	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
362	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
363	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
364	static FNVMXEXITHANDLER hmR0VmxExitHlt;
365	static FNVMXEXITHANDLER hmR0VmxExitInvd;
366	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
367	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
368	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
369	static FNVMXEXITHANDLER hmR0VmxExitRsm;
370	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
371	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
372	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
373	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
374	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
375	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
376	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
377	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
378	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
379	static FNVMXEXITHANDLER hmR0VmxExitMwait;
380	static FNVMXEXITHANDLER hmR0VmxExitMtf;
381	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
382	static FNVMXEXITHANDLER hmR0VmxExitPause;
383	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
384	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
385	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
386	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
387	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
389	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
390	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
391	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
392	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
393	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
394	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
395	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
396	/** @} */
397
398	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
399	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
400	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
406
407	/*******************************************************************************
408	* Global Variables *
409	*******************************************************************************/
410	#ifdef HMVMX_USE_FUNCTION_TABLE
411
412	/**
413	* VMX_EXIT dispatch table.
414	*/
415	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
416	{
417	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
418	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
419	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
420	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
421	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
422	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
423	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
424	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
425	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
426	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
427	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
428	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
429	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
430	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
431	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
432	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
433	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
434	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
435	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
436	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
437	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
438	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
439	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
440	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
441	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
442	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
443	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
444	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
445	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
446	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
447	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
448	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
449	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
450	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
451	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
452	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
453	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
454	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
455	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
456	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
457	/* 40 UNDEFINED */ hmR0VmxExitPause,
458	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
459	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
460	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
461	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
462	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
463	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
464	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
465	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
466	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
467	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
468	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
469	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
470	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
471	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
472	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
473	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
474	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
475	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
476	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
477	};
478	#endif /* HMVMX_USE_FUNCTION_TABLE */
479
480	#ifdef VBOX_STRICT
481	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
482	{
483	/* 0 */ "(Not Used)",
484	/* 1 */ "VMCALL executed in VMX root operation.",
485	/* 2 */ "VMCLEAR with invalid physical address.",
486	/* 3 */ "VMCLEAR with VMXON pointer.",
487	/* 4 */ "VMLAUNCH with non-clear VMCS.",
488	/* 5 */ "VMRESUME with non-launched VMCS.",
489	/* 6 */ "VMRESUME after VMXOFF",
490	/* 7 */ "VM entry with invalid control fields.",
491	/* 8 */ "VM entry with invalid host state fields.",
492	/* 9 */ "VMPTRLD with invalid physical address.",
493	/* 10 */ "VMPTRLD with VMXON pointer.",
494	/* 11 */ "VMPTRLD with incorrect revision identifier.",
495	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
496	/* 13 */ "VMWRITE to read-only VMCS component.",
497	/* 14 */ "(Not Used)",
498	/* 15 */ "VMXON executed in VMX root operation.",
499	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
500	/* 17 */ "VM entry with non-launched executing VMCS.",
501	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
502	/* 19 */ "VMCALL with non-clear VMCS.",
503	/* 20 */ "VMCALL with invalid VM-exit control fields.",
504	/* 21 */ "(Not Used)",
505	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
506	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
507	/* 24 */ "VMCALL with invalid SMM-monitor features.",
508	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
509	/* 26 */ "VM entry with events blocked by MOV SS.",
510	/* 27 */ "(Not Used)",
511	/* 28 */ "Invalid operand to INVEPT/INVVPID."
512	};
513	#endif /* VBOX_STRICT */
514
515
516
517	/**
518	* Updates the VM's last error record. If there was a VMX instruction error,
519	* reads the error data from the VMCS and updates VCPU's last error record as
520	* well.
521	*
522	* @param pVM Pointer to the VM.
523	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
524	* VERR_VMX_UNABLE_TO_START_VM or
525	* VERR_VMX_INVALID_VMCS_FIELD).
526	* @param rc The error code.
527	*/
528	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
529	{
530	AssertPtr(pVM);
531	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
532	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
533	{
534	AssertPtrReturnVoid(pVCpu);
535	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
536	}
537	pVM->hm.s.lLastError = rc;
538	}
539
540
541	/**
542	* Reads the VM-entry interruption-information field from the VMCS into the VMX
543	* transient structure.
544	*
545	* @returns VBox status code.
546	* @param pVmxTransient Pointer to the VMX transient structure.
547	*
548	* @remarks No-long-jump zone!!!
549	*/
550	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
551	{
552	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
553	AssertRCReturn(rc, rc);
554	return VINF_SUCCESS;
555	}
556
557
558	/**
559	* Reads the VM-entry exception error code field from the VMCS into
560	* the VMX transient structure.
561	*
562	* @returns VBox status code.
563	* @param pVmxTransient Pointer to the VMX transient structure.
564	*
565	* @remarks No-long-jump zone!!!
566	*/
567	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
568	{
569	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
570	AssertRCReturn(rc, rc);
571	return VINF_SUCCESS;
572	}
573
574
575	/**
576	* Reads the VM-entry exception error code field from the VMCS into
577	* the VMX transient structure.
578	*
579	* @returns VBox status code.
580	* @param pVCpu Pointer to the VMCPU.
581	* @param pVmxTransient Pointer to the VMX transient structure.
582	*
583	* @remarks No-long-jump zone!!!
584	*/
585	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
586	{
587	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
588	AssertRCReturn(rc, rc);
589	return VINF_SUCCESS;
590	}
591
592
593	/**
594	* Reads the VM-exit interruption-information field from the VMCS into the VMX
595	* transient structure.
596	*
597	* @returns VBox status code.
598	* @param pVCpu Pointer to the VMCPU.
599	* @param pVmxTransient Pointer to the VMX transient structure.
600	*/
601	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
602	{
603	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
604	{
605	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
606	AssertRCReturn(rc, rc);
607	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
608	}
609	return VINF_SUCCESS;
610	}
611
612
613	/**
614	* Reads the VM-exit interruption error code from the VMCS into the VMX
615	* transient structure.
616	*
617	* @returns VBox status code.
618	* @param pVCpu Pointer to the VMCPU.
619	* @param pVmxTransient Pointer to the VMX transient structure.
620	*/
621	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
622	{
623	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
624	{
625	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
626	AssertRCReturn(rc, rc);
627	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
628	}
629	return VINF_SUCCESS;
630	}
631
632
633	/**
634	* Reads the VM-exit instruction length field from the VMCS into the VMX
635	* transient structure.
636	*
637	* @returns VBox status code.
638	* @param pVCpu Pointer to the VMCPU.
639	* @param pVmxTransient Pointer to the VMX transient structure.
640	*/
641	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
642	{
643	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
644	{
645	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
646	AssertRCReturn(rc, rc);
647	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
648	}
649	return VINF_SUCCESS;
650	}
651
652
653	/**
654	* Reads the VM-exit instruction-information field from the VMCS into
655	* the VMX transient structure.
656	*
657	* @returns VBox status code.
658	* @param pVCpu The cross context per CPU structure.
659	* @param pVmxTransient Pointer to the VMX transient structure.
660	*/
661	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
662	{
663	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664	{
665	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666	AssertRCReturn(rc, rc);
667	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668	}
669	return VINF_SUCCESS;
670	}
671
672
673	/**
674	* Reads the exit qualification from the VMCS into the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVCpu Pointer to the VMCPU.
678	* @param pVmxTransient Pointer to the VMX transient structure.
679	*/
680	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
681	{
682	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
683	{
684	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
685	AssertRCReturn(rc, rc);
686	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
687	}
688	return VINF_SUCCESS;
689	}
690
691
692	/**
693	* Reads the IDT-vectoring information field from the VMCS into the VMX
694	* transient structure.
695	*
696	* @returns VBox status code.
697	* @param pVmxTransient Pointer to the VMX transient structure.
698	*
699	* @remarks No-long-jump zone!!!
700	*/
701	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
702	{
703	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
704	{
705	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
706	AssertRCReturn(rc, rc);
707	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
708	}
709	return VINF_SUCCESS;
710	}
711
712
713	/**
714	* Reads the IDT-vectoring error code from the VMCS into the VMX
715	* transient structure.
716	*
717	* @returns VBox status code.
718	* @param pVmxTransient Pointer to the VMX transient structure.
719	*/
720	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
721	{
722	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
723	{
724	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
725	AssertRCReturn(rc, rc);
726	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
727	}
728	return VINF_SUCCESS;
729	}
730
731
732	/**
733	* Enters VMX root mode operation on the current CPU.
734	*
735	* @returns VBox status code.
736	* @param pVM Pointer to the VM (optional, can be NULL, after
737	* a resume).
738	* @param HCPhysCpuPage Physical address of the VMXON region.
739	* @param pvCpuPage Pointer to the VMXON region.
740	*/
741	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
742	{
743	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
744	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
745	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
746
747	if (pVM)
748	{
749	/* Write the VMCS revision dword to the VMXON region. */
750	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
751	}
752
753	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
754	RTCCUINTREG uEflags = ASMIntDisableFlags();
755
756	/* Enable the VMX bit in CR4 if necessary. */
757	RTCCUINTREG uCr4 = ASMGetCR4();
758	if (!(uCr4 & X86_CR4_VMXE))
759	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
760
761	/* Enter VMX root mode. */
762	int rc = VMXEnable(HCPhysCpuPage);
763	if (RT_FAILURE(rc))
764	ASMSetCR4(uCr4);
765
766	/* Restore interrupts. */
767	ASMSetFlags(uEflags);
768	return rc;
769	}
770
771
772	/**
773	* Exits VMX root mode operation on the current CPU.
774	*
775	* @returns VBox status code.
776	*/
777	static int hmR0VmxLeaveRootMode(void)
778	{
779	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
780
781	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
782	RTCCUINTREG uEflags = ASMIntDisableFlags();
783
784	/* If we're for some reason not in VMX root mode, then don't leave it. */
785	RTCCUINTREG uHostCR4 = ASMGetCR4();
786
787	int rc;
788	if (uHostCR4 & X86_CR4_VMXE)
789	{
790	/* Exit VMX root mode and clear the VMX bit in CR4. */
791	VMXDisable();
792	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
793	rc = VINF_SUCCESS;
794	}
795	else
796	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
797
798	/* Restore interrupts. */
799	ASMSetFlags(uEflags);
800	return rc;
801	}
802
803
804	/**
805	* Allocates and maps one physically contiguous page. The allocated page is
806	* zero'd out. (Used by various VT-x structures).
807	*
808	* @returns IPRT status code.
809	* @param pMemObj Pointer to the ring-0 memory object.
810	* @param ppVirt Where to store the virtual address of the
811	* allocation.
812	* @param pPhys Where to store the physical address of the
813	* allocation.
814	*/
815	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
816	{
817	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
818	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
819	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
820
821	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
822	if (RT_FAILURE(rc))
823	return rc;
824	ppVirt = RTR0MemObjAddress(pMemObj);
825	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
826	ASMMemZero32(*ppVirt, PAGE_SIZE);
827	return VINF_SUCCESS;
828	}
829
830
831	/**
832	* Frees and unmaps an allocated physical page.
833	*
834	* @param pMemObj Pointer to the ring-0 memory object.
835	* @param ppVirt Where to re-initialize the virtual address of
836	* allocation as 0.
837	* @param pHCPhys Where to re-initialize the physical address of the
838	* allocation as 0.
839	*/
840	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
841	{
842	AssertPtr(pMemObj);
843	AssertPtr(ppVirt);
844	AssertPtr(pHCPhys);
845	if (*pMemObj != NIL_RTR0MEMOBJ)
846	{
847	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
848	AssertRC(rc);
849	*pMemObj = NIL_RTR0MEMOBJ;
850	*ppVirt = 0;
851	*pHCPhys = 0;
852	}
853	}
854
855
856	/**
857	* Worker function to free VT-x related structures.
858	*
859	* @returns IPRT status code.
860	* @param pVM Pointer to the VM.
861	*/
862	static void hmR0VmxStructsFree(PVM pVM)
863	{
864	for (VMCPUID i = 0; i < pVM->cCpus; i++)
865	{
866	PVMCPU pVCpu = &pVM->aCpus[i];
867	AssertPtr(pVCpu);
868
869	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872	#endif
873
874	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
875	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
876
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
878	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
879	}
880
881	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
882	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
883	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
884	#endif
885	}
886
887
888	/**
889	* Worker function to allocate VT-x related VM structures.
890	*
891	* @returns IPRT status code.
892	* @param pVM Pointer to the VM.
893	*/
894	static int hmR0VmxStructsAlloc(PVM pVM)
895	{
896	/*
897	* Initialize members up-front so we can cleanup properly on allocation failure.
898	*/
899	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
900	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
901	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
902	pVM->hm.s.vmx.HCPhys##a_Name = 0;
903
904	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
905	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
908
909	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
910	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
911	#endif
912	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
913
914	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
915	for (VMCPUID i = 0; i < pVM->cCpus; i++)
916	{
917	PVMCPU pVCpu = &pVM->aCpus[i];
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
921	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
922	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
923	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
924	#endif
925	}
926	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
927	#undef VMXLOCAL_INIT_VM_MEMOBJ
928
929	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
930	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
931	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
932	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
933
934	/*
935	* Allocate all the VT-x structures.
936	*/
937	int rc = VINF_SUCCESS;
938	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
939	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
940	if (RT_FAILURE(rc))
941	goto cleanup;
942	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
943	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
944	#endif
945
946	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
947	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
948	{
949	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
950	&pVM->hm.s.vmx.HCPhysApicAccess);
951	if (RT_FAILURE(rc))
952	goto cleanup;
953	}
954
955	/*
956	* Initialize per-VCPU VT-x structures.
957	*/
958	for (VMCPUID i = 0; i < pVM->cCpus; i++)
959	{
960	PVMCPU pVCpu = &pVM->aCpus[i];
961	AssertPtr(pVCpu);
962
963	/* Allocate the VM control structure (VMCS). */
964	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
965	if (RT_FAILURE(rc))
966	goto cleanup;
967
968	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
969	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
970	{
971	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
972	&pVCpu->hm.s.vmx.HCPhysVirtApic);
973	if (RT_FAILURE(rc))
974	goto cleanup;
975	}
976
977	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
978	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
979	{
980	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
981	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
982	if (RT_FAILURE(rc))
983	goto cleanup;
984	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
985	}
986
987	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
988	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
989	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
990	if (RT_FAILURE(rc))
991	goto cleanup;
992
993	/* Allocate the VM-exit MSR-load page for the host MSRs. */
994	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
995	if (RT_FAILURE(rc))
996	goto cleanup;
997	#endif
998	}
999
1000	return VINF_SUCCESS;
1001
1002	cleanup:
1003	hmR0VmxStructsFree(pVM);
1004	return rc;
1005	}
1006
1007
1008	/**
1009	* Does global VT-x initialization (called during module initialization).
1010	*
1011	* @returns VBox status code.
1012	*/
1013	VMMR0DECL(int) VMXR0GlobalInit(void)
1014	{
1015	#ifdef HMVMX_USE_FUNCTION_TABLE
1016	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1017	# ifdef VBOX_STRICT
1018	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1019	Assert(g_apfnVMExitHandlers[i]);
1020	# endif
1021	#endif
1022	return VINF_SUCCESS;
1023	}
1024
1025
1026	/**
1027	* Does global VT-x termination (called during module termination).
1028	*/
1029	VMMR0DECL(void) VMXR0GlobalTerm()
1030	{
1031	/* Nothing to do currently. */
1032	}
1033
1034
1035	/**
1036	* Sets up and activates VT-x on the current CPU.
1037	*
1038	* @returns VBox status code.
1039	* @param pCpu Pointer to the global CPU info struct.
1040	* @param pVM Pointer to the VM (can be NULL after a host resume
1041	* operation).
1042	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1043	* fEnabledByHost is true).
1044	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1045	* @a fEnabledByHost is true).
1046	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1047	* enable VT-x on the host.
1048	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1049	*/
1050	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1051	void *pvMsrs)
1052	{
1053	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1054	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1055	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1056
1057	/* Enable VT-x if it's not already enabled by the host. */
1058	if (!fEnabledByHost)
1059	{
1060	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1061	if (RT_FAILURE(rc))
1062	return rc;
1063	}
1064
1065	/*
1066	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1067	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1068	*/
1069	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1070	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1071	{
1072	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1073	pCpu->fFlushAsidBeforeUse = false;
1074	}
1075	else
1076	pCpu->fFlushAsidBeforeUse = true;
1077
1078	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1079	++pCpu->cTlbFlushes;
1080
1081	return VINF_SUCCESS;
1082	}
1083
1084
1085	/**
1086	* Deactivates VT-x on the current CPU.
1087	*
1088	* @returns VBox status code.
1089	* @param pCpu Pointer to the global CPU info struct.
1090	* @param pvCpuPage Pointer to the VMXON region.
1091	* @param HCPhysCpuPage Physical address of the VMXON region.
1092	*
1093	* @remarks This function should never be called when SUPR0EnableVTx() or
1094	* similar was used to enable VT-x on the host.
1095	*/
1096	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1097	{
1098	NOREF(pCpu);
1099	NOREF(pvCpuPage);
1100	NOREF(HCPhysCpuPage);
1101
1102	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1103	return hmR0VmxLeaveRootMode();
1104	}
1105
1106
1107	/**
1108	* Sets the permission bits for the specified MSR in the MSR bitmap.
1109	*
1110	* @param pVCpu Pointer to the VMCPU.
1111	* @param uMSR The MSR value.
1112	* @param enmRead Whether reading this MSR causes a VM-exit.
1113	* @param enmWrite Whether writing this MSR causes a VM-exit.
1114	*/
1115	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1116	{
1117	int32_t iBit;
1118	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1119
1120	/*
1121	* Layout:
1122	* 0x000 - 0x3ff - Low MSR read bits
1123	* 0x400 - 0x7ff - High MSR read bits
1124	* 0x800 - 0xbff - Low MSR write bits
1125	* 0xc00 - 0xfff - High MSR write bits
1126	*/
1127	if (uMsr <= 0x00001FFF)
1128	iBit = uMsr;
1129	else if ( uMsr >= 0xC0000000
1130	&& uMsr <= 0xC0001FFF)
1131	{
1132	iBit = (uMsr - 0xC0000000);
1133	pbMsrBitmap += 0x400;
1134	}
1135	else
1136	{
1137	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1138	return;
1139	}
1140
1141	Assert(iBit <= 0x1fff);
1142	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1143	ASMBitSet(pbMsrBitmap, iBit);
1144	else
1145	ASMBitClear(pbMsrBitmap, iBit);
1146
1147	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1148	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1149	else
1150	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1151	}
1152
1153
1154	/**
1155	* Flushes the TLB using EPT.
1156	*
1157	* @returns VBox status code.
1158	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1159	* enmFlush).
1160	* @param enmFlush Type of flush.
1161	*
1162	* @remarks Caller is responsible for making sure this function is called only
1163	* when NestedPaging is supported and providing @a enmFlush that is
1164	* supported by the CPU.
1165	* @remarks Can be called with interrupts disabled.
1166	*/
1167	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1168	{
1169	uint64_t au64Descriptor[2];
1170	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1171	au64Descriptor[0] = 0;
1172	else
1173	{
1174	Assert(pVCpu);
1175	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1176	}
1177	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1178
1179	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1180	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1181	rc));
1182	if ( RT_SUCCESS(rc)
1183	&& pVCpu)
1184	{
1185	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1186	}
1187	}
1188
1189
1190	/**
1191	* Flushes the TLB using VPID.
1192	*
1193	* @returns VBox status code.
1194	* @param pVM Pointer to the VM.
1195	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1196	* enmFlush).
1197	* @param enmFlush Type of flush.
1198	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1199	* on @a enmFlush).
1200	*
1201	* @remarks Can be called with interrupts disabled.
1202	*/
1203	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1204	{
1205	AssertPtr(pVM);
1206	Assert(pVM->hm.s.vmx.fVpid);
1207
1208	uint64_t au64Descriptor[2];
1209	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1210	{
1211	au64Descriptor[0] = 0;
1212	au64Descriptor[1] = 0;
1213	}
1214	else
1215	{
1216	AssertPtr(pVCpu);
1217	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1218	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1219	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1220	au64Descriptor[1] = GCPtr;
1221	}
1222
1223	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1224	AssertMsg(rc == VINF_SUCCESS,
1225	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1226	if ( RT_SUCCESS(rc)
1227	&& pVCpu)
1228	{
1229	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1230	}
1231	}
1232
1233
1234	/**
1235	* Invalidates a guest page by guest virtual address. Only relevant for
1236	* EPT/VPID, otherwise there is nothing really to invalidate.
1237	*
1238	* @returns VBox status code.
1239	* @param pVM Pointer to the VM.
1240	* @param pVCpu Pointer to the VMCPU.
1241	* @param GCVirt Guest virtual address of the page to invalidate.
1242	*/
1243	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1244	{
1245	AssertPtr(pVM);
1246	AssertPtr(pVCpu);
1247	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1248
1249	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1250	if (!fFlushPending)
1251	{
1252	/*
1253	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1254	* See @bugref{6043} and @bugref{6177}.
1255	*
1256	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1257	* function maybe called in a loop with individual addresses.
1258	*/
1259	if (pVM->hm.s.vmx.fVpid)
1260	{
1261	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1262	{
1263	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1264	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1265	}
1266	else
1267	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1268	}
1269	else if (pVM->hm.s.fNestedPaging)
1270	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1271	}
1272
1273	return VINF_SUCCESS;
1274	}
1275
1276
1277	/**
1278	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1279	* otherwise there is nothing really to invalidate.
1280	*
1281	* @returns VBox status code.
1282	* @param pVM Pointer to the VM.
1283	* @param pVCpu Pointer to the VMCPU.
1284	* @param GCPhys Guest physical address of the page to invalidate.
1285	*/
1286	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1287	{
1288	LogFlowFunc(("%RGp\n", GCPhys));
1289
1290	/*
1291	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1292	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1293	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1294	*/
1295	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1296	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1297	return VINF_SUCCESS;
1298	}
1299
1300
1301	/**
1302	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1303	* case where neither EPT nor VPID is supported by the CPU.
1304	*
1305	* @param pVM Pointer to the VM.
1306	* @param pVCpu Pointer to the VMCPU.
1307	* @param pCpu Pointer to the global HM struct.
1308	*
1309	* @remarks Called with interrupts disabled.
1310	*/
1311	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1312	{
1313	AssertPtr(pVCpu);
1314	AssertPtr(pCpu);
1315	NOREF(pVM);
1316
1317	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1318
1319	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1320	#if 0
1321	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1322	pVCpu->hm.s.TlbShootdown.cPages = 0;
1323	#endif
1324
1325	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1326	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1327	pVCpu->hm.s.fForceTLBFlush = false;
1328	return;
1329	}
1330
1331
1332	/**
1333	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1334	*
1335	* @param pVM Pointer to the VM.
1336	* @param pVCpu Pointer to the VMCPU.
1337	* @param pCpu Pointer to the global HM CPU struct.
1338	* @remarks All references to "ASID" in this function pertains to "VPID" in
1339	* Intel's nomenclature. The reason is, to avoid confusion in compare
1340	* statements since the host-CPU copies are named "ASID".
1341	*
1342	* @remarks Called with interrupts disabled.
1343	*/
1344	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1345	{
1346	#ifdef VBOX_WITH_STATISTICS
1347	bool fTlbFlushed = false;
1348	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1349	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1350	if (!fTlbFlushed) \
1351	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1352	} while (0)
1353	#else
1354	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1355	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1356	#endif
1357
1358	AssertPtr(pVM);
1359	AssertPtr(pCpu);
1360	AssertPtr(pVCpu);
1361	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1362	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1363	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1364
1365
1366	/*
1367	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1368	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1369	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1370	*/
1371	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1372	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1373	{
1374	++pCpu->uCurrentAsid;
1375	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1376	{
1377	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1378	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1379	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1380	}
1381
1382	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1383	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1384	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1385
1386	/*
1387	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1388	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1389	*/
1390	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1391	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1392	HMVMX_SET_TAGGED_TLB_FLUSHED();
1393	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1394	}
1395
1396	/* Check for explicit TLB shootdowns. */
1397	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1398	{
1399	/*
1400	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1401	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1402	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1403	* but not guest-physical mappings.
1404	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1405	*/
1406	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1407	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1408	HMVMX_SET_TAGGED_TLB_FLUSHED();
1409	}
1410
1411	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1412	* where it is commented out. Support individual entry flushing
1413	* someday. */
1414	#if 0
1415	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1416	{
1417	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1418
1419	/*
1420	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1421	* as supported by the CPU.
1422	*/
1423	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1424	{
1425	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1426	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1427	}
1428	else
1429	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1430
1431	HMVMX_SET_TAGGED_TLB_FLUSHED();
1432	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1433	pVCpu->hm.s.TlbShootdown.cPages = 0;
1434	}
1435	#endif
1436
1437	pVCpu->hm.s.fForceTLBFlush = false;
1438
1439	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1440
1441	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1442	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1443	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1444	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1445	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1446	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1447	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1448	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1449
1450	/* Update VMCS with the VPID. */
1451	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1452	AssertRC(rc);
1453
1454	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1455	}
1456
1457
1458	/**
1459	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1460	*
1461	* @returns VBox status code.
1462	* @param pVM Pointer to the VM.
1463	* @param pVCpu Pointer to the VMCPU.
1464	* @param pCpu Pointer to the global HM CPU struct.
1465	*
1466	* @remarks Called with interrupts disabled.
1467	*/
1468	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1469	{
1470	AssertPtr(pVM);
1471	AssertPtr(pVCpu);
1472	AssertPtr(pCpu);
1473	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1474	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1475
1476	/*
1477	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1478	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1479	*/
1480	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1481	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1482	{
1483	pVCpu->hm.s.fForceTLBFlush = true;
1484	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1485	}
1486
1487	/* Check for explicit TLB shootdown flushes. */
1488	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1489	{
1490	pVCpu->hm.s.fForceTLBFlush = true;
1491	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1492	}
1493
1494	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1495	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1496
1497	if (pVCpu->hm.s.fForceTLBFlush)
1498	{
1499	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1500	pVCpu->hm.s.fForceTLBFlush = false;
1501	}
1502	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1503	* where it is commented out. Support individual entry flushing
1504	* someday. */
1505	#if 0
1506	else
1507	{
1508	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1509	{
1510	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1511	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1512	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1513	}
1514	else
1515	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1516
1517	pVCpu->hm.s.TlbShootdown.cPages = 0;
1518	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1519	}
1520	#endif
1521	}
1522
1523
1524	/**
1525	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1526	*
1527	* @returns VBox status code.
1528	* @param pVM Pointer to the VM.
1529	* @param pVCpu Pointer to the VMCPU.
1530	* @param pCpu Pointer to the global HM CPU struct.
1531	*
1532	* @remarks Called with interrupts disabled.
1533	*/
1534	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1535	{
1536	AssertPtr(pVM);
1537	AssertPtr(pVCpu);
1538	AssertPtr(pCpu);
1539	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1540	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1541
1542	/*
1543	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1544	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1545	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1546	*/
1547	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1548	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1549	{
1550	pVCpu->hm.s.fForceTLBFlush = true;
1551	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1552	}
1553
1554	/* Check for explicit TLB shootdown flushes. */
1555	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1556	{
1557	/*
1558	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1559	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1560	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1561	*/
1562	pVCpu->hm.s.fForceTLBFlush = true;
1563	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1564	}
1565
1566	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1567	if (pVCpu->hm.s.fForceTLBFlush)
1568	{
1569	++pCpu->uCurrentAsid;
1570	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1571	{
1572	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1573	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1574	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1575	}
1576
1577	pVCpu->hm.s.fForceTLBFlush = false;
1578	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1579	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1580	if (pCpu->fFlushAsidBeforeUse)
1581	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1582	}
1583	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1584	* where it is commented out. Support individual entry flushing
1585	* someday. */
1586	#if 0
1587	else
1588	{
1589	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1590	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1591	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1592	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1593
1594	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1595	{
1596	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1597	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1598	{
1599	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1600	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1601	}
1602	else
1603	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1604
1605	pVCpu->hm.s.TlbShootdown.cPages = 0;
1606	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1607	}
1608	else
1609	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1610	}
1611	#endif
1612
1613	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1614	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1615	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1616	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1617	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1618	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1619
1620	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1621	AssertRC(rc);
1622	}
1623
1624
1625	/**
1626	* Flushes the guest TLB entry based on CPU capabilities.
1627	*
1628	* @param pVCpu Pointer to the VMCPU.
1629	* @param pCpu Pointer to the global HM CPU struct.
1630	*/
1631	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1632	{
1633	PVM pVM = pVCpu->CTX_SUFF(pVM);
1634	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1635	{
1636	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1637	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1638	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1639	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1640	default:
1641	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1642	break;
1643	}
1644	}
1645
1646
1647	/**
1648	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1649	* TLB entries from the host TLB before VM-entry.
1650	*
1651	* @returns VBox status code.
1652	* @param pVM Pointer to the VM.
1653	*/
1654	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1655	{
1656	/*
1657	* Determine optimal flush type for Nested Paging.
1658	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1659	* guest execution (see hmR3InitFinalizeR0()).
1660	*/
1661	if (pVM->hm.s.fNestedPaging)
1662	{
1663	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1664	{
1665	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1666	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1667	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1668	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1669	else
1670	{
1671	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1672	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1673	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1674	}
1675
1676	/* Make sure the write-back cacheable memory type for EPT is supported. */
1677	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1678	{
1679	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1680	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1681	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1682	}
1683	}
1684	else
1685	{
1686	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1687	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1688	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1689	}
1690	}
1691
1692	/*
1693	* Determine optimal flush type for VPID.
1694	*/
1695	if (pVM->hm.s.vmx.fVpid)
1696	{
1697	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1698	{
1699	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1700	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1701	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1702	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1703	else
1704	{
1705	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1706	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1707	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1708	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1709	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1710	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1711	pVM->hm.s.vmx.fVpid = false;
1712	}
1713	}
1714	else
1715	{
1716	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1717	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1718	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1719	pVM->hm.s.vmx.fVpid = false;
1720	}
1721	}
1722
1723	/*
1724	* Setup the handler for flushing tagged-TLBs.
1725	*/
1726	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1727	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1728	else if (pVM->hm.s.fNestedPaging)
1729	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1730	else if (pVM->hm.s.vmx.fVpid)
1731	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1732	else
1733	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1734	return VINF_SUCCESS;
1735	}
1736
1737
1738	/**
1739	* Sets up pin-based VM-execution controls in the VMCS.
1740	*
1741	* @returns VBox status code.
1742	* @param pVM Pointer to the VM.
1743	* @param pVCpu Pointer to the VMCPU.
1744	*/
1745	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1746	{
1747	AssertPtr(pVM);
1748	AssertPtr(pVCpu);
1749
1750	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1751	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1752
1753	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1754	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1755	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1756
1757	/* Enable the VMX preemption timer. */
1758	if (pVM->hm.s.vmx.fUsePreemptTimer)
1759	{
1760	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1761	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1762	}
1763
1764	if ((val & zap) != val)
1765	{
1766	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1767	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1768	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1769	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1770	}
1771
1772	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1773	AssertRCReturn(rc, rc);
1774
1775	/* Update VCPU with the currently set pin-based VM-execution controls. */
1776	pVCpu->hm.s.vmx.u32PinCtls = val;
1777	return rc;
1778	}
1779
1780
1781	/**
1782	* Sets up processor-based VM-execution controls in the VMCS.
1783	*
1784	* @returns VBox status code.
1785	* @param pVM Pointer to the VM.
1786	* @param pVMCPU Pointer to the VMCPU.
1787	*/
1788	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1789	{
1790	AssertPtr(pVM);
1791	AssertPtr(pVCpu);
1792
1793	int rc = VERR_INTERNAL_ERROR_5;
1794	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1795	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1796
1797	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1798	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1799	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1800	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1801	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1802	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1803	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1804
1805	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1806	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1807	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1808	{
1809	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1810	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1811	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1812	}
1813
1814	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1815	if (!pVM->hm.s.fNestedPaging)
1816	{
1817	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1818	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1819	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1820	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1821	}
1822
1823	/* Use TPR shadowing if supported by the CPU. */
1824	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1825	{
1826	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1827	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1828	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1829	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1830	AssertRCReturn(rc, rc);
1831
1832	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1833	/* CR8 writes causes a VM-exit based on TPR threshold. */
1834	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1835	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1836	}
1837	else
1838	{
1839	/*
1840	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
1841	* Set this control only for 64-bit guests.
1842	*/
1843	if (pVM->hm.s.fAllow64BitGuests)
1844	{
1845	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1846	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1847	}
1848	}
1849
1850	/* Use MSR-bitmaps if supported by the CPU. */
1851	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1852	{
1853	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1854
1855	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1856	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1857	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1858	AssertRCReturn(rc, rc);
1859
1860	/*
1861	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1862	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1863	*/
1864	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1865	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1866	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1867	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1868	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1869	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1870	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1871	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1872	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1873	}
1874
1875	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1876	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1877	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1878
1879	if ((val & zap) != val)
1880	{
1881	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1882	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1883	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1884	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1885	}
1886
1887	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1888	AssertRCReturn(rc, rc);
1889
1890	/* Update VCPU with the currently set processor-based VM-execution controls. */
1891	pVCpu->hm.s.vmx.u32ProcCtls = val;
1892
1893	/*
1894	* Secondary processor-based VM-execution controls.
1895	*/
1896	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1897	{
1898	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1899	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1900
1901	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1902	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1903
1904	if (pVM->hm.s.fNestedPaging)
1905	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1906	else
1907	{
1908	/*
1909	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1910	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1911	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1912	*/
1913	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1914	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1915	}
1916
1917	if (pVM->hm.s.vmx.fVpid)
1918	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1919
1920	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1921	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1922
1923	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1924	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1925	* done dynamically. */
1926	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1927	{
1928	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1929	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1930	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1931	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1932	AssertRCReturn(rc, rc);
1933	}
1934
1935	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1936	{
1937	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1938	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1939	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1940	}
1941
1942	if ((val & zap) != val)
1943	{
1944	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1945	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1946	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1947	}
1948
1949	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1950	AssertRCReturn(rc, rc);
1951
1952	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1953	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1954	}
1955	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1956	{
1957	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1958	"available\n"));
1959	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1960	}
1961
1962	return VINF_SUCCESS;
1963	}
1964
1965
1966	/**
1967	* Sets up miscellaneous (everything other than Pin & Processor-based
1968	* VM-execution) control fields in the VMCS.
1969	*
1970	* @returns VBox status code.
1971	* @param pVM Pointer to the VM.
1972	* @param pVCpu Pointer to the VMCPU.
1973	*/
1974	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1975	{
1976	AssertPtr(pVM);
1977	AssertPtr(pVCpu);
1978
1979	int rc = VERR_GENERAL_FAILURE;
1980
1981	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1982	#if 0
1983	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
1984	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1985	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1986
1987	/*
1988	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1989	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1990	* We thus use the exception bitmap to control it rather than use both.
1991	*/
1992	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1993	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1994
1995	/** @todo Explore possibility of using IO-bitmaps. */
1996	/* All IO & IOIO instructions cause VM-exits. */
1997	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1998	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1999
2000	/* Initialize the MSR-bitmap area. */
2001	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2002	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2003	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2004	#endif
2005
2006	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2007	/* Setup MSR autoloading/storing. */
2008	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2009	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2010	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2011	AssertRCReturn(rc, rc);
2012	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2013	AssertRCReturn(rc, rc);
2014
2015	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2016	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2017	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2018	AssertRCReturn(rc, rc);
2019	#endif
2020
2021	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2022	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2023	AssertRCReturn(rc, rc);
2024
2025	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2026	#if 0
2027	/* Setup debug controls */
2028	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2029	AssertRCReturn(rc, rc);
2030	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2031	AssertRCReturn(rc, rc);
2032	#endif
2033
2034	return rc;
2035	}
2036
2037
2038	/**
2039	* Sets up the initial exception bitmap in the VMCS based on static conditions
2040	* (i.e. conditions that cannot ever change after starting the VM).
2041	*
2042	* @returns VBox status code.
2043	* @param pVM Pointer to the VM.
2044	* @param pVCpu Pointer to the VMCPU.
2045	*/
2046	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2047	{
2048	AssertPtr(pVM);
2049	AssertPtr(pVCpu);
2050
2051	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2052
2053	uint32_t u32XcptBitmap = 0;
2054
2055	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2056	if (!pVM->hm.s.fNestedPaging)
2057	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2058
2059	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2060	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2061	AssertRCReturn(rc, rc);
2062	return rc;
2063	}
2064
2065
2066	/**
2067	* Sets up the initial guest-state mask. The guest-state mask is consulted
2068	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2069	* for the nested virtualization case (as it would cause a VM-exit).
2070	*
2071	* @param pVCpu Pointer to the VMCPU.
2072	*/
2073	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2074	{
2075	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2076	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2077	return VINF_SUCCESS;
2078	}
2079
2080
2081	/**
2082	* Does per-VM VT-x initialization.
2083	*
2084	* @returns VBox status code.
2085	* @param pVM Pointer to the VM.
2086	*/
2087	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2088	{
2089	LogFlowFunc(("pVM=%p\n", pVM));
2090
2091	int rc = hmR0VmxStructsAlloc(pVM);
2092	if (RT_FAILURE(rc))
2093	{
2094	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2095	return rc;
2096	}
2097
2098	return VINF_SUCCESS;
2099	}
2100
2101
2102	/**
2103	* Does per-VM VT-x termination.
2104	*
2105	* @returns VBox status code.
2106	* @param pVM Pointer to the VM.
2107	*/
2108	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2109	{
2110	LogFlowFunc(("pVM=%p\n", pVM));
2111
2112	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2113	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2114	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2115	#endif
2116	hmR0VmxStructsFree(pVM);
2117	return VINF_SUCCESS;
2118	}
2119
2120
2121	/**
2122	* Sets up the VM for execution under VT-x.
2123	* This function is only called once per-VM during initialization.
2124	*
2125	* @returns VBox status code.
2126	* @param pVM Pointer to the VM.
2127	*/
2128	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2129	{
2130	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2131	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2132
2133	LogFlowFunc(("pVM=%p\n", pVM));
2134
2135	/*
2136	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2137	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2138	*/
2139	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2140	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2141	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2142	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2143	{
2144	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2145	return VERR_INTERNAL_ERROR;
2146	}
2147
2148	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2149	/*
2150	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2151	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2152	*/
2153	if ( pVM->hm.s.fAllow64BitGuests
2154	&& !HMVMX_IS_64BIT_HOST_MODE())
2155	{
2156	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2157	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2158	}
2159	#endif
2160
2161	/* Initialize these always, see hmR3InitFinalizeR0().*/
2162	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2163	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2164
2165	/* Setup the tagged-TLB flush handlers. */
2166	int rc = hmR0VmxSetupTaggedTlb(pVM);
2167	if (RT_FAILURE(rc))
2168	{
2169	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2170	return rc;
2171	}
2172
2173	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2174	{
2175	PVMCPU pVCpu = &pVM->aCpus[i];
2176	AssertPtr(pVCpu);
2177	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2178
2179	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2180	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2181
2182	/* Set revision dword at the beginning of the VMCS structure. */
2183	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2184
2185	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2186	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2187	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2188	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2189
2190	/* Load this VMCS as the current VMCS. */
2191	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2192	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2193	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2194
2195	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2196	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2197	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2198
2199	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2200	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2201	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2202
2203	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2204	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2205	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2206
2207	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2208	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2209	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2210
2211	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2212	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2213	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2214
2215	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2216	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2217	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2218	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2219	#endif
2220
2221	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2222	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2223	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2224	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2225
2226	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2227
2228	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2229	}
2230
2231	return VINF_SUCCESS;
2232	}
2233
2234
2235	/**
2236	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2237	* the VMCS.
2238	*
2239	* @returns VBox status code.
2240	* @param pVM Pointer to the VM.
2241	* @param pVCpu Pointer to the VMCPU.
2242	*/
2243	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2244	{
2245	RTCCUINTREG uReg = ASMGetCR0();
2246	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2247	AssertRCReturn(rc, rc);
2248
2249	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2250	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2251	if (HMVMX_IS_64BIT_HOST_MODE())
2252	{
2253	uint64_t uRegCR3 = HMR0Get64bitCR3();
2254	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2255	}
2256	else
2257	#endif
2258	{
2259	uReg = ASMGetCR3();
2260	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2261	}
2262	AssertRCReturn(rc, rc);
2263
2264	uReg = ASMGetCR4();
2265	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2266	AssertRCReturn(rc, rc);
2267	return rc;
2268	}
2269
2270
2271	#if HC_ARCH_BITS == 64
2272	/**
2273	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2274	* requirements. See hmR0VmxSaveHostSegmentRegs().
2275	*/
2276	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2277	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2278	{ \
2279	bool fValidSelector = true; \
2280	if ((selValue) & X86_SEL_LDT) \
2281	{ \
2282	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2283	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2284	} \
2285	if (fValidSelector) \
2286	{ \
2287	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2288	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2289	} \
2290	(selValue) = 0; \
2291	}
2292	#endif
2293
2294
2295	/**
2296	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2297	* the host-state area in the VMCS.
2298	*
2299	* @returns VBox status code.
2300	* @param pVM Pointer to the VM.
2301	* @param pVCpu Pointer to the VMCPU.
2302	*/
2303	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2304	{
2305	int rc = VERR_INTERNAL_ERROR_5;
2306
2307	/*
2308	* Host DS, ES, FS and GS segment registers.
2309	*/
2310	#if HC_ARCH_BITS == 64
2311	RTSEL uSelDS = ASMGetDS();
2312	RTSEL uSelES = ASMGetES();
2313	RTSEL uSelFS = ASMGetFS();
2314	RTSEL uSelGS = ASMGetGS();
2315	#else
2316	RTSEL uSelDS = 0;
2317	RTSEL uSelES = 0;
2318	RTSEL uSelFS = 0;
2319	RTSEL uSelGS = 0;
2320	#endif
2321
2322	/* Recalculate which host-state bits need to be manually restored. */
2323	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2324
2325	/*
2326	* Host CS and SS segment registers.
2327	*/
2328	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2329	RTSEL uSelCS;
2330	RTSEL uSelSS;
2331	if (HMVMX_IS_64BIT_HOST_MODE())
2332	{
2333	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2334	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2335	}
2336	else
2337	{
2338	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2339	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2340	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2341	}
2342	#else
2343	RTSEL uSelCS = ASMGetCS();
2344	RTSEL uSelSS = ASMGetSS();
2345	#endif
2346
2347	/*
2348	* Host TR segment register.
2349	*/
2350	RTSEL uSelTR = ASMGetTR();
2351
2352	#if HC_ARCH_BITS == 64
2353	/*
2354	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2355	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2356	*/
2357	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2358	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2359	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2360	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2361	# undef VMXLOCAL_ADJUST_HOST_SEG
2362	#endif
2363
2364	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2365	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2366	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2367	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2368	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2369	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2370	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2371	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2372	Assert(uSelCS);
2373	Assert(uSelTR);
2374
2375	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2376	#if 0
2377	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2378	Assert(uSelSS != 0);
2379	#endif
2380
2381	/* Write these host selector fields into the host-state area in the VMCS. */
2382	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2383	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2384	#if HC_ARCH_BITS == 64
2385	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2386	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2387	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2388	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2389	#endif
2390	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2391
2392	/*
2393	* Host GDTR and IDTR.
2394	*/
2395	RTGDTR Gdtr;
2396	RT_ZERO(Gdtr);
2397	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2398	if (HMVMX_IS_64BIT_HOST_MODE())
2399	{
2400	X86XDTR64 Gdtr64;
2401	X86XDTR64 Idtr64;
2402	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2403	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2404	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2405
2406	Gdtr.cbGdt = Gdtr64.cb;
2407	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2408	}
2409	else
2410	#endif
2411	{
2412	RTIDTR Idtr;
2413	ASMGetGDTR(&Gdtr);
2414	ASMGetIDTR(&Idtr);
2415	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2416	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2417
2418	#if HC_ARCH_BITS == 64
2419	/*
2420	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2421	* maximum limit (0xffff) on every VM-exit.
2422	*/
2423	if (Gdtr.cbGdt != 0xffff)
2424	{
2425	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2426	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2427	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2428	}
2429
2430	/*
2431	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2432	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2433	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2434	*/
2435	if (Idtr.cbIdt < 0x0fff)
2436	{
2437	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2438	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2439	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2440	}
2441	#endif
2442	}
2443
2444	/*
2445	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2446	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2447	*/
2448	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2449	{
2450	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2451	return VERR_VMX_INVALID_HOST_STATE;
2452	}
2453
2454	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2455	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2456	if (HMVMX_IS_64BIT_HOST_MODE())
2457	{
2458	/* We need the 64-bit TR base for hybrid darwin. */
2459	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2460	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2461	}
2462	else
2463	#endif
2464	{
2465	uintptr_t uTRBase;
2466	#if HC_ARCH_BITS == 64
2467	uTRBase = X86DESC64_BASE(pDesc);
2468
2469	/*
2470	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2471	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2472	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2473	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2474	*
2475	* [1] See Intel spec. 3.5 "System Descriptor Types".
2476	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2477	*/
2478	Assert(pDesc->System.u4Type == 11);
2479	if ( pDesc->System.u16LimitLow != 0x67
2480	\|\| pDesc->System.u4LimitHigh)
2481	{
2482	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2483	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2484
2485	/* Store the GDTR here as we need it while restoring TR. */
2486	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2487	}
2488	#else
2489	uTRBase = X86DESC_BASE(pDesc);
2490	#endif
2491	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2492	}
2493	AssertRCReturn(rc, rc);
2494
2495	/*
2496	* Host FS base and GS base.
2497	*/
2498	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2499	if (HMVMX_IS_64BIT_HOST_MODE())
2500	{
2501	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2502	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2503	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2504	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2505
2506	# if HC_ARCH_BITS == 64
2507	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2508	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2509	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2510	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2511	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2512	# endif
2513	}
2514	#endif
2515	return rc;
2516	}
2517
2518
2519	/**
2520	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2521	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2522	* the host after every successful VM exit.
2523	*
2524	* @returns VBox status code.
2525	* @param pVM Pointer to the VM.
2526	* @param pVCpu Pointer to the VMCPU.
2527	*/
2528	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2529	{
2530	AssertPtr(pVCpu);
2531	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2532
2533	int rc = VINF_SUCCESS;
2534	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2535	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2536	uint32_t cHostMsrs = 0;
2537	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2538
2539	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2540	{
2541	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2542
2543	# if HC_ARCH_BITS == 64
2544	/* Paranoia. 64-bit code requires these bits to be set always. */
2545	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2546
2547	/*
2548	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2549	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2550	* some reason (e.g. allow transparent reads) we would activate the code below.
2551	*/
2552	# if 0
2553	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2554	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2555	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2556	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2557	if (CPUMIsGuestInLongMode(pVCpu))
2558	{
2559	uint64_t u64GuestEfer;
2560	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2561	AssertRC(rc);
2562
2563	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2564	{
2565	pHostMsr->u32Msr = MSR_K6_EFER;
2566	pHostMsr->u32Reserved = 0;
2567	pHostMsr->u64Value = u64HostEfer;
2568	pHostMsr++; cHostMsrs++;
2569	}
2570	}
2571	# endif
2572	# else /* HC_ARCH_BITS != 64 */
2573	pHostMsr->u32Msr = MSR_K6_EFER;
2574	pHostMsr->u32Reserved = 0;
2575	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2576	if (CPUMIsGuestInLongMode(pVCpu))
2577	{
2578	/* Must match the EFER value in our 64 bits switcher. */
2579	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2580	}
2581	else
2582	# endif
2583	pHostMsr->u64Value = u64HostEfer;
2584	pHostMsr++; cHostMsrs++;
2585	# endif /* HC_ARCH_BITS == 64 */
2586	}
2587
2588	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2589	if (HMVMX_IS_64BIT_HOST_MODE())
2590	{
2591	pHostMsr->u32Msr = MSR_K6_STAR;
2592	pHostMsr->u32Reserved = 0;
2593	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2594	pHostMsr++; cHostMsrs++;
2595	pHostMsr->u32Msr = MSR_K8_LSTAR;
2596	pHostMsr->u32Reserved = 0;
2597	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2598	pHostMsr++; cHostMsrs++;
2599	pHostMsr->u32Msr = MSR_K8_SF_MASK;
2600	pHostMsr->u32Reserved = 0;
2601	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2602	pHostMsr++; cHostMsrs++;
2603	pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2604	pHostMsr->u32Reserved = 0;
2605	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2606	pHostMsr++; cHostMsrs++;
2607	}
2608	# endif
2609
2610	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2611	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2612	{
2613	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2614	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2615	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2616	}
2617
2618	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2619	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2620
2621	/*
2622	* Host Sysenter MSRs.
2623	*/
2624	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2625	AssertRCReturn(rc, rc);
2626	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2627	if (HMVMX_IS_64BIT_HOST_MODE())
2628	{
2629	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2630	AssertRCReturn(rc, rc);
2631	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2632	}
2633	else
2634	{
2635	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2636	AssertRCReturn(rc, rc);
2637	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2638	}
2639	#elif HC_ARCH_BITS == 32
2640	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2641	AssertRCReturn(rc, rc);
2642	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2643	#else
2644	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2645	AssertRCReturn(rc, rc);
2646	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2647	#endif
2648	AssertRCReturn(rc, rc);
2649
2650	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2651	* hmR0VmxSetupExitCtls() !! */
2652	return rc;
2653	}
2654
2655
2656	/**
2657	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2658	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2659	* controls".
2660	*
2661	* @returns VBox status code.
2662	* @param pVCpu Pointer to the VMCPU.
2663	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2664	* out-of-sync. Make sure to update the required fields
2665	* before using them.
2666	*
2667	* @remarks No-long-jump zone!!!
2668	*/
2669	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2670	{
2671	int rc = VINF_SUCCESS;
2672	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
2673	{
2674	PVM pVM = pVCpu->CTX_SUFF(pVM);
2675	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2676	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2677
2678	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2679	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2680
2681	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2682	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2683	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2684	else
2685	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2686
2687	/*
2688	* The following should -not- be set (since we're not in SMM mode):
2689	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2690	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2691	*/
2692
2693	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2694	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2695	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2696
2697	if ((val & zap) != val)
2698	{
2699	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2700	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2701	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2702	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2703	}
2704
2705	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2706	AssertRCReturn(rc, rc);
2707
2708	/* Update VCPU with the currently set VM-exit controls. */
2709	pVCpu->hm.s.vmx.u32EntryCtls = val;
2710	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
2711	}
2712	return rc;
2713	}
2714
2715
2716	/**
2717	* Sets up the VM-exit controls in the VMCS.
2718	*
2719	* @returns VBox status code.
2720	* @param pVM Pointer to the VM.
2721	* @param pVCpu Pointer to the VMCPU.
2722	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2723	* out-of-sync. Make sure to update the required fields
2724	* before using them.
2725	*
2726	* @remarks requires EFER.
2727	*/
2728	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2729	{
2730	int rc = VINF_SUCCESS;
2731	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
2732	{
2733	PVM pVM = pVCpu->CTX_SUFF(pVM);
2734	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2735	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2736
2737	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2738	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2739
2740	/*
2741	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2742	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2743	*/
2744	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2745	if (HMVMX_IS_64BIT_HOST_MODE())
2746	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2747	else
2748	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2749	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2750	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2751	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2752	else
2753	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2754	#endif
2755
2756	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2757	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2758
2759	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2760	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2761	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2762	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2763	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2764
2765	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2766	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2767
2768	if ((val & zap) != val)
2769	{
2770	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2771	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2772	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2773	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2774	}
2775
2776	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2777	AssertRCReturn(rc, rc);
2778
2779	/* Update VCPU with the currently set VM-exit controls. */
2780	pVCpu->hm.s.vmx.u32ExitCtls = val;
2781	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
2782	}
2783	return rc;
2784	}
2785
2786
2787	/**
2788	* Loads the guest APIC and related state.
2789	*
2790	* @returns VBox status code.
2791	* @param pVM Pointer to the VM.
2792	* @param pVCpu Pointer to the VMCPU.
2793	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2794	* out-of-sync. Make sure to update the required fields
2795	* before using them.
2796	*/
2797	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2798	{
2799	int rc = VINF_SUCCESS;
2800	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
2801	{
2802	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2803	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2804	{
2805	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2806
2807	bool fPendingIntr = false;
2808	uint8_t u8Tpr = 0;
2809	uint8_t u8PendingIntr = 0;
2810	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2811	AssertRCReturn(rc, rc);
2812
2813	/*
2814	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2815	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2816	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2817	* the interrupt when we VM-exit for other reasons.
2818	*/
2819	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2820	uint32_t u32TprThreshold = 0;
2821	if (fPendingIntr)
2822	{
2823	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2824	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2825	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2826	if (u8PendingPriority <= u8TprPriority)
2827	u32TprThreshold = u8PendingPriority;
2828	else
2829	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2830	}
2831	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2832
2833	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2834	AssertRCReturn(rc, rc);
2835	}
2836
2837	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
2838	}
2839	return rc;
2840	}
2841
2842
2843	/**
2844	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2845	*
2846	* @returns Guest's interruptibility-state.
2847	* @param pVCpu Pointer to the VMCPU.
2848	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2849	* out-of-sync. Make sure to update the required fields
2850	* before using them.
2851	*
2852	* @remarks No-long-jump zone!!!
2853	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2854	*/
2855	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2856	{
2857	/*
2858	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2859	* inhibit interrupts or clear any existing interrupt-inhibition.
2860	*/
2861	uint32_t uIntrState = 0;
2862	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2863	{
2864	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2865	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2866	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2867	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2868	{
2869	/*
2870	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2871	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2872	*/
2873	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2874	}
2875	else if (pMixedCtx->eflags.Bits.u1IF)
2876	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2877	else
2878	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2879	}
2880	return uIntrState;
2881	}
2882
2883
2884	/**
2885	* Loads the guest's interruptibility-state into the guest-state area in the
2886	* VMCS.
2887	*
2888	* @returns VBox status code.
2889	* @param pVCpu Pointer to the VMCPU.
2890	* @param uIntrState The interruptibility-state to set.
2891	*/
2892	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2893	{
2894	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2895	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2896	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2897	AssertRCReturn(rc, rc);
2898	return rc;
2899	}
2900
2901
2902	/**
2903	* Loads the guest's RIP into the guest-state area in the VMCS.
2904	*
2905	* @returns VBox status code.
2906	* @param pVCpu Pointer to the VMCPU.
2907	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2908	* out-of-sync. Make sure to update the required fields
2909	* before using them.
2910	*
2911	* @remarks No-long-jump zone!!!
2912	*/
2913	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2914	{
2915	int rc = VINF_SUCCESS;
2916	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
2917	{
2918	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2919	AssertRCReturn(rc, rc);
2920
2921	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
2922	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
2923	}
2924	return rc;
2925	}
2926
2927
2928	/**
2929	* Loads the guest's RSP into the guest-state area in the VMCS.
2930	*
2931	* @returns VBox status code.
2932	* @param pVCpu Pointer to the VMCPU.
2933	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2934	* out-of-sync. Make sure to update the required fields
2935	* before using them.
2936	*
2937	* @remarks No-long-jump zone!!!
2938	*/
2939	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2940	{
2941	int rc = VINF_SUCCESS;
2942	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
2943	{
2944	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2945	AssertRCReturn(rc, rc);
2946
2947	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
2948	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2949	}
2950	return rc;
2951	}
2952
2953
2954	/**
2955	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2956	*
2957	* @returns VBox status code.
2958	* @param pVCpu Pointer to the VMCPU.
2959	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2960	* out-of-sync. Make sure to update the required fields
2961	* before using them.
2962	*
2963	* @remarks No-long-jump zone!!!
2964	*/
2965	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2966	{
2967	int rc = VINF_SUCCESS;
2968	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
2969	{
2970	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2971	Let us assert it as such and use 32-bit VMWRITE. */
2972	Assert(!(pMixedCtx->rflags.u64 >> 32));
2973	X86EFLAGS Eflags = pMixedCtx->eflags;
2974	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2975	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2976
2977	/*
2978	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2979	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2980	*/
2981	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2982	{
2983	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2984	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2985	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2986	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2987	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2988	}
2989
2990	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2991	AssertRCReturn(rc, rc);
2992
2993	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
2994	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2995	}
2996	return rc;
2997	}
2998
2999
3000	/**
3001	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3002	*
3003	* @returns VBox status code.
3004	* @param pVCpu Pointer to the VMCPU.
3005	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3006	* out-of-sync. Make sure to update the required fields
3007	* before using them.
3008	*
3009	* @remarks No-long-jump zone!!!
3010	*/
3011	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3012	{
3013	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3014	AssertRCReturn(rc, rc);
3015	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3016	AssertRCReturn(rc, rc);
3017	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3018	AssertRCReturn(rc, rc);
3019	return rc;
3020	}
3021
3022
3023	/**
3024	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3025	* CR0 is partially shared with the host and we have to consider the FPU bits.
3026	*
3027	* @returns VBox status code.
3028	* @param pVM Pointer to the VM.
3029	* @param pVCpu Pointer to the VMCPU.
3030	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3031	* out-of-sync. Make sure to update the required fields
3032	* before using them.
3033	*
3034	* @remarks No-long-jump zone!!!
3035	*/
3036	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3037	{
3038	/*
3039	* Guest CR0.
3040	* Guest FPU.
3041	*/
3042	int rc = VINF_SUCCESS;
3043	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3044	{
3045	Assert(!(pMixedCtx->cr0 >> 32));
3046	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3047	PVM pVM = pVCpu->CTX_SUFF(pVM);
3048
3049	/* The guest's view (read access) of its CR0 is unblemished. */
3050	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3051	AssertRCReturn(rc, rc);
3052	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3053
3054	/* Setup VT-x's view of the guest CR0. */
3055	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3056	if (pVM->hm.s.fNestedPaging)
3057	{
3058	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3059	{
3060	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3061	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3062	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3063	}
3064	else
3065	{
3066	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3067	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3068	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3069	}
3070
3071	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3072	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3073	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3074
3075	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3076	AssertRCReturn(rc, rc);
3077	}
3078	else
3079	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3080
3081	/*
3082	* Guest FPU bits.
3083	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3084	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3085	*/
3086	u32GuestCR0 \|= X86_CR0_NE;
3087	bool fInterceptNM = false;
3088	if (CPUMIsGuestFPUStateActive(pVCpu))
3089	{
3090	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3091	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3092	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3093	}
3094	else
3095	{
3096	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3097	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3098	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3099	}
3100
3101	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3102	bool fInterceptMF = false;
3103	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3104	fInterceptMF = true;
3105
3106	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3107	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3108	{
3109	Assert(PDMVmmDevHeapIsEnabled(pVM));
3110	Assert(pVM->hm.s.vmx.pRealModeTSS);
3111	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3112	fInterceptNM = true;
3113	fInterceptMF = true;
3114	}
3115	else
3116	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3117
3118	if (fInterceptNM)
3119	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3120	else
3121	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3122
3123	if (fInterceptMF)
3124	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3125	else
3126	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3127
3128	/* Additional intercepts for debugging, define these yourself explicitly. */
3129	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3130	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3131	\| RT_BIT(X86_XCPT_BP)
3132	\| RT_BIT(X86_XCPT_DB)
3133	\| RT_BIT(X86_XCPT_DE)
3134	\| RT_BIT(X86_XCPT_NM)
3135	\| RT_BIT(X86_XCPT_UD)
3136	\| RT_BIT(X86_XCPT_NP)
3137	\| RT_BIT(X86_XCPT_SS)
3138	\| RT_BIT(X86_XCPT_GP)
3139	\| RT_BIT(X86_XCPT_PF)
3140	\| RT_BIT(X86_XCPT_MF)
3141	;
3142	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3143	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3144	#endif
3145
3146	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3147
3148	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3149	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3150	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3151	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3152	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3153	else
3154	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3155
3156	u32GuestCR0 \|= uSetCR0;
3157	u32GuestCR0 &= uZapCR0;
3158	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3159
3160	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3161	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3162	AssertRCReturn(rc, rc);
3163	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3164	AssertRCReturn(rc, rc);
3165	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3166
3167	/*
3168	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3169	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3170	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3171	*/
3172	uint32_t u32CR0Mask = 0;
3173	u32CR0Mask = X86_CR0_PE
3174	\| X86_CR0_NE
3175	\| X86_CR0_WP
3176	\| X86_CR0_PG
3177	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3178	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3179	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3180
3181	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3182	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3183	* and @bugref{6944}. */
3184	#if 0
3185	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3186	u32CR0Mask &= ~X86_CR0_PE;
3187	#endif
3188	if (pVM->hm.s.fNestedPaging)
3189	u32CR0Mask &= ~X86_CR0_WP;
3190
3191	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3192	if (fInterceptNM)
3193	{
3194	u32CR0Mask \|= X86_CR0_TS
3195	\| X86_CR0_MP;
3196	}
3197
3198	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3199	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3200	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3201	AssertRCReturn(rc, rc);
3202	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3203
3204	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3205	}
3206	return rc;
3207	}
3208
3209
3210	/**
3211	* Loads the guest control registers (CR3, CR4) into the guest-state area
3212	* in the VMCS.
3213	*
3214	* @returns VBox status code.
3215	* @param pVM Pointer to the VM.
3216	* @param pVCpu Pointer to the VMCPU.
3217	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3218	* out-of-sync. Make sure to update the required fields
3219	* before using them.
3220	*
3221	* @remarks No-long-jump zone!!!
3222	*/
3223	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3224	{
3225	int rc = VINF_SUCCESS;
3226	PVM pVM = pVCpu->CTX_SUFF(pVM);
3227
3228	/*
3229	* Guest CR2.
3230	* It's always loaded in the assembler code. Nothing to do here.
3231	*/
3232
3233	/*
3234	* Guest CR3.
3235	*/
3236	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3237	{
3238	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3239	if (pVM->hm.s.fNestedPaging)
3240	{
3241	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3242
3243	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3244	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3245	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3246	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3247
3248	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3249	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3250	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3251
3252	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3253	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3254	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3255	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3256
3257	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3258	AssertRCReturn(rc, rc);
3259	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3260
3261	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3262	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3263	{
3264	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3265	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3266	{
3267	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3268	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3269	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3270	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3271	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3272	}
3273
3274	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3275	have Unrestricted Execution to handle the guest when it's not using paging. */
3276	GCPhysGuestCR3 = pMixedCtx->cr3;
3277	}
3278	else
3279	{
3280	/*
3281	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3282	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3283	* EPT takes care of translating it to host-physical addresses.
3284	*/
3285	RTGCPHYS GCPhys;
3286	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3287	Assert(PDMVmmDevHeapIsEnabled(pVM));
3288
3289	/* We obtain it here every time as the guest could have relocated this PCI region. */
3290	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3291	AssertRCReturn(rc, rc);
3292
3293	GCPhysGuestCR3 = GCPhys;
3294	}
3295
3296	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3297	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3298	}
3299	else
3300	{
3301	/* Non-nested paging case, just use the hypervisor's CR3. */
3302	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3303
3304	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3305	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3306	}
3307	AssertRCReturn(rc, rc);
3308
3309	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3310	}
3311
3312	/*
3313	* Guest CR4.
3314	*/
3315	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3316	{
3317	Assert(!(pMixedCtx->cr4 >> 32));
3318	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3319
3320	/* The guest's view of its CR4 is unblemished. */
3321	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3322	AssertRCReturn(rc, rc);
3323	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3324
3325	/* Setup VT-x's view of the guest CR4. */
3326	/*
3327	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3328	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3329	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3330	*/
3331	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3332	{
3333	Assert(pVM->hm.s.vmx.pRealModeTSS);
3334	Assert(PDMVmmDevHeapIsEnabled(pVM));
3335	u32GuestCR4 &= ~X86_CR4_VME;
3336	}
3337
3338	if (pVM->hm.s.fNestedPaging)
3339	{
3340	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3341	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3342	{
3343	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3344	u32GuestCR4 \|= X86_CR4_PSE;
3345	/* Our identity mapping is a 32 bits page directory. */
3346	u32GuestCR4 &= ~X86_CR4_PAE;
3347	}
3348	/* else use guest CR4.*/
3349	}
3350	else
3351	{
3352	/*
3353	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3354	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3355	*/
3356	switch (pVCpu->hm.s.enmShadowMode)
3357	{
3358	case PGMMODE_REAL: /* Real-mode. */
3359	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3360	case PGMMODE_32_BIT: /* 32-bit paging. */
3361	{
3362	u32GuestCR4 &= ~X86_CR4_PAE;
3363	break;
3364	}
3365
3366	case PGMMODE_PAE: /* PAE paging. */
3367	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3368	{
3369	u32GuestCR4 \|= X86_CR4_PAE;
3370	break;
3371	}
3372
3373	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3374	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3375	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3376	break;
3377	#endif
3378	default:
3379	AssertFailed();
3380	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3381	}
3382	}
3383
3384	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3385	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3386	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3387	u32GuestCR4 \|= uSetCR4;
3388	u32GuestCR4 &= uZapCR4;
3389
3390	/* Write VT-x's view of the guest CR4 into the VMCS. */
3391	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3392	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3393	AssertRCReturn(rc, rc);
3394
3395	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3396	uint32_t u32CR4Mask = 0;
3397	u32CR4Mask = X86_CR4_VME
3398	\| X86_CR4_PAE
3399	\| X86_CR4_PGE
3400	\| X86_CR4_PSE
3401	\| X86_CR4_VMXE;
3402	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3403	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3404	AssertRCReturn(rc, rc);
3405
3406	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3407	}
3408	return rc;
3409	}
3410
3411
3412	/**
3413	* Loads the guest debug registers into the guest-state area in the VMCS.
3414	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3415	*
3416	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3417	*
3418	* @returns VBox status code.
3419	* @param pVCpu Pointer to the VMCPU.
3420	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3421	* out-of-sync. Make sure to update the required fields
3422	* before using them.
3423	*
3424	* @remarks No-long-jump zone!!!
3425	*/
3426	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3427	{
3428	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3429	return VINF_SUCCESS;
3430
3431	#ifdef VBOX_STRICT
3432	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3433	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3434	{
3435	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3436	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3437	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3438	}
3439	#endif
3440
3441	int rc;
3442	PVM pVM = pVCpu->CTX_SUFF(pVM);
3443	bool fInterceptDB = false;
3444	bool fInterceptMovDRx = false;
3445	if (pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu))
3446	{
3447	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3448	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3449	{
3450	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3451	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3452	AssertRCReturn(rc, rc);
3453	Assert(fInterceptDB == false);
3454	}
3455	else
3456	{
3457	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3458	pVCpu->hm.s.fClearTrapFlag = true;
3459	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3460	fInterceptDB = true;
3461	}
3462	}
3463
3464	if (fInterceptDB \|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3465	{
3466	/*
3467	* Use the combined guest and host DRx values found in the hypervisor
3468	* register set because the debugger has breakpoints active or someone
3469	* is single stepping on the host side without a monitor trap flag.
3470	*
3471	* Note! DBGF expects a clean DR6 state before executing guest code.
3472	*/
3473	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3474	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3475	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3476	{
3477	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3478	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3479	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3480	}
3481	else
3482	#endif
3483	if (!CPUMIsHyperDebugStateActive(pVCpu))
3484	{
3485	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3486	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3487	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3488	}
3489
3490	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3491	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3492	AssertRCReturn(rc, rc);
3493
3494	pVCpu->hm.s.fUsingHyperDR7 = true;
3495	fInterceptDB = true;
3496	fInterceptMovDRx = true;
3497	}
3498	else
3499	{
3500	/*
3501	* If the guest has enabled debug registers, we need to load them prior to
3502	* executing guest code so they'll trigger at the right time.
3503	*/
3504	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3505	{
3506	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3507	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3508	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3509	{
3510	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3511	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3512	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3513	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3514	}
3515	else
3516	#endif
3517	if (CPUMIsGuestDebugStateActive(pVCpu))
3518	{
3519	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3520	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3521	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3522	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3523	}
3524	}
3525	/*
3526	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3527	* must intercept #DB in order to maintain a correct DR6 guest value.
3528	*/
3529	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3530	else if ( ( CPUMIsGuestInLongModeEx(pMixedCtx)
3531	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3532	\|\| !CPUMIsGuestDebugStateActive(pVCpu))
3533	#else
3534	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3535	#endif
3536	{
3537	fInterceptMovDRx = true;
3538	fInterceptDB = true;
3539	}
3540
3541	/* Update guest DR7. */
3542	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3543	AssertRCReturn(rc, rc);
3544
3545	pVCpu->hm.s.fUsingHyperDR7 = false;
3546	}
3547
3548	/*
3549	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3550	*/
3551	if (fInterceptDB)
3552	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3553	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3554	{
3555	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3556	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3557	#endif
3558	}
3559	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3560	AssertRCReturn(rc, rc);
3561
3562	/*
3563	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3564	*/
3565	if (fInterceptMovDRx)
3566	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3567	else
3568	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3569	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3570	AssertRCReturn(rc, rc);
3571
3572	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3573	return VINF_SUCCESS;
3574	}
3575
3576
3577	#ifdef VBOX_STRICT
3578	/**
3579	* Strict function to validate segment registers.
3580	*
3581	* @remarks ASSUMES CR0 is up to date.
3582	*/
3583	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3584	{
3585	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3586	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3587	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3588	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3589	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3590	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3591	{
3592	/* Protected mode checks */
3593	/* CS */
3594	Assert(pCtx->cs.Attr.n.u1Present);
3595	Assert(!(pCtx->cs.Attr.u & 0xf00));
3596	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3597	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3598	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3599	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3600	\|\| (pCtx->cs.Attr.n.u1Granularity));
3601	/* CS cannot be loaded with NULL in protected mode. */
3602	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3603	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3604	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3605	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3606	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3607	else
3608	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3609	/* SS */
3610	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3611	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3612	if ( !(pCtx->cr0 & X86_CR0_PE)
3613	\|\| pCtx->cs.Attr.n.u4Type == 3)
3614	{
3615	Assert(!pCtx->ss.Attr.n.u2Dpl);
3616	}
3617	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3618	{
3619	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3620	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3621	Assert(pCtx->ss.Attr.n.u1Present);
3622	Assert(!(pCtx->ss.Attr.u & 0xf00));
3623	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3624	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3625	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3626	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3627	\|\| (pCtx->ss.Attr.n.u1Granularity));
3628	}
3629	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3630	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3631	{
3632	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3633	Assert(pCtx->ds.Attr.n.u1Present);
3634	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3635	Assert(!(pCtx->ds.Attr.u & 0xf00));
3636	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3637	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3638	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3639	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3640	\|\| (pCtx->ds.Attr.n.u1Granularity));
3641	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3642	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3643	}
3644	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3645	{
3646	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3647	Assert(pCtx->es.Attr.n.u1Present);
3648	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3649	Assert(!(pCtx->es.Attr.u & 0xf00));
3650	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3651	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3652	\|\| !(pCtx->es.Attr.n.u1Granularity));
3653	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3654	\|\| (pCtx->es.Attr.n.u1Granularity));
3655	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3656	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3657	}
3658	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3659	{
3660	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3661	Assert(pCtx->fs.Attr.n.u1Present);
3662	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3663	Assert(!(pCtx->fs.Attr.u & 0xf00));
3664	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3665	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3666	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3667	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3668	\|\| (pCtx->fs.Attr.n.u1Granularity));
3669	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3670	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3671	}
3672	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3673	{
3674	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3675	Assert(pCtx->gs.Attr.n.u1Present);
3676	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3677	Assert(!(pCtx->gs.Attr.u & 0xf00));
3678	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3679	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3680	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3681	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3682	\|\| (pCtx->gs.Attr.n.u1Granularity));
3683	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3684	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3685	}
3686	/* 64-bit capable CPUs. */
3687	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3688	Assert(!(pCtx->cs.u64Base >> 32));
3689	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3690	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3691	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3692	# endif
3693	}
3694	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3695	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3696	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3697	{
3698	/* Real and v86 mode checks. */
3699	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3700	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3701	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3702	{
3703	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3704	}
3705	else
3706	{
3707	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3708	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3709	}
3710
3711	/* CS */
3712	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3713	Assert(pCtx->cs.u32Limit == 0xffff);
3714	Assert(u32CSAttr == 0xf3);
3715	/* SS */
3716	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3717	Assert(pCtx->ss.u32Limit == 0xffff);
3718	Assert(u32SSAttr == 0xf3);
3719	/* DS */
3720	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3721	Assert(pCtx->ds.u32Limit == 0xffff);
3722	Assert(u32DSAttr == 0xf3);
3723	/* ES */
3724	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3725	Assert(pCtx->es.u32Limit == 0xffff);
3726	Assert(u32ESAttr == 0xf3);
3727	/* FS */
3728	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3729	Assert(pCtx->fs.u32Limit == 0xffff);
3730	Assert(u32FSAttr == 0xf3);
3731	/* GS */
3732	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3733	Assert(pCtx->gs.u32Limit == 0xffff);
3734	Assert(u32GSAttr == 0xf3);
3735	/* 64-bit capable CPUs. */
3736	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3737	Assert(!(pCtx->cs.u64Base >> 32));
3738	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3739	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3740	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3741	# endif
3742	}
3743	}
3744	#endif /* VBOX_STRICT */
3745
3746
3747	/**
3748	* Writes a guest segment register into the guest-state area in the VMCS.
3749	*
3750	* @returns VBox status code.
3751	* @param pVCpu Pointer to the VMCPU.
3752	* @param idxSel Index of the selector in the VMCS.
3753	* @param idxLimit Index of the segment limit in the VMCS.
3754	* @param idxBase Index of the segment base in the VMCS.
3755	* @param idxAccess Index of the access rights of the segment in the VMCS.
3756	* @param pSelReg Pointer to the segment selector.
3757	* @param pCtx Pointer to the guest-CPU context.
3758	*
3759	* @remarks No-long-jump zone!!!
3760	*/
3761	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3762	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3763	{
3764	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3765	AssertRCReturn(rc, rc);
3766	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3767	AssertRCReturn(rc, rc);
3768	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3769	AssertRCReturn(rc, rc);
3770
3771	uint32_t u32Access = pSelReg->Attr.u;
3772	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3773	{
3774	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3775	u32Access = 0xf3;
3776	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3777	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3778	}
3779	else
3780	{
3781	/*
3782	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3783	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3784	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3785	* loaded in protected-mode have their attribute as 0.
3786	*/
3787	if (!u32Access)
3788	u32Access = X86DESCATTR_UNUSABLE;
3789	}
3790
3791	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3792	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3793	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3794
3795	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3796	AssertRCReturn(rc, rc);
3797	return rc;
3798	}
3799
3800
3801	/**
3802	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3803	* into the guest-state area in the VMCS.
3804	*
3805	* @returns VBox status code.
3806	* @param pVM Pointer to the VM.
3807	* @param pVCPU Pointer to the VMCPU.
3808	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3809	* out-of-sync. Make sure to update the required fields
3810	* before using them.
3811	*
3812	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
3813	* @remarks No-long-jump zone!!!
3814	*/
3815	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3816	{
3817	int rc = VERR_INTERNAL_ERROR_5;
3818	PVM pVM = pVCpu->CTX_SUFF(pVM);
3819
3820	/*
3821	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3822	*/
3823	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
3824	{
3825	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3826	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3827	{
3828	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3829	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3830	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3831	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3832	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3833	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3834	}
3835
3836	#ifdef VBOX_WITH_REM
3837	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3838	{
3839	Assert(pVM->hm.s.vmx.pRealModeTSS);
3840	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3841	if ( pVCpu->hm.s.vmx.fWasInRealMode
3842	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3843	{
3844	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3845	in real-mode (e.g. OpenBSD 4.0) */
3846	REMFlushTBs(pVM);
3847	Log4(("Load: Switch to protected mode detected!\n"));
3848	pVCpu->hm.s.vmx.fWasInRealMode = false;
3849	}
3850	}
3851	#endif
3852	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3853	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3854	AssertRCReturn(rc, rc);
3855	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3856	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3857	AssertRCReturn(rc, rc);
3858	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3859	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3860	AssertRCReturn(rc, rc);
3861	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3862	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3863	AssertRCReturn(rc, rc);
3864	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3865	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3866	AssertRCReturn(rc, rc);
3867	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3868	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3869	AssertRCReturn(rc, rc);
3870
3871	#ifdef VBOX_STRICT
3872	/* Validate. */
3873	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3874	#endif
3875
3876	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
3877	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3878	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3879	}
3880
3881	/*
3882	* Guest TR.
3883	*/
3884	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
3885	{
3886	/*
3887	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3888	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3889	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3890	*/
3891	uint16_t u16Sel = 0;
3892	uint32_t u32Limit = 0;
3893	uint64_t u64Base = 0;
3894	uint32_t u32AccessRights = 0;
3895
3896	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3897	{
3898	u16Sel = pMixedCtx->tr.Sel;
3899	u32Limit = pMixedCtx->tr.u32Limit;
3900	u64Base = pMixedCtx->tr.u64Base;
3901	u32AccessRights = pMixedCtx->tr.Attr.u;
3902	}
3903	else
3904	{
3905	Assert(pVM->hm.s.vmx.pRealModeTSS);
3906	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3907
3908	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3909	RTGCPHYS GCPhys;
3910	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3911	AssertRCReturn(rc, rc);
3912
3913	X86DESCATTR DescAttr;
3914	DescAttr.u = 0;
3915	DescAttr.n.u1Present = 1;
3916	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3917
3918	u16Sel = 0;
3919	u32Limit = HM_VTX_TSS_SIZE;
3920	u64Base = GCPhys; /* in real-mode phys = virt. */
3921	u32AccessRights = DescAttr.u;
3922	}
3923
3924	/* Validate. */
3925	Assert(!(u16Sel & RT_BIT(2)));
3926	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3927	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3928	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3929	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3930	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3931	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3932	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3933	Assert( (u32Limit & 0xfff) == 0xfff
3934	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3935	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3936	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3937
3938	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3939	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3940	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3941	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3942
3943	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
3944	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3945	}
3946
3947	/*
3948	* Guest GDTR.
3949	*/
3950	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
3951	{
3952	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3953	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3954
3955	/* Validate. */
3956	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3957
3958	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
3959	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3960	}
3961
3962	/*
3963	* Guest LDTR.
3964	*/
3965	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
3966	{
3967	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3968	uint32_t u32Access = 0;
3969	if (!pMixedCtx->ldtr.Attr.u)
3970	u32Access = X86DESCATTR_UNUSABLE;
3971	else
3972	u32Access = pMixedCtx->ldtr.Attr.u;
3973
3974	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3975	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3976	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3977	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3978
3979	/* Validate. */
3980	if (!(u32Access & X86DESCATTR_UNUSABLE))
3981	{
3982	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3983	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3984	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3985	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3986	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3987	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3988	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3989	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3990	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3991	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3992	}
3993
3994	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
3995	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3996	}
3997
3998	/*
3999	* Guest IDTR.
4000	*/
4001	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4002	{
4003	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4004	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4005
4006	/* Validate. */
4007	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4008
4009	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4010	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4011	}
4012
4013	return VINF_SUCCESS;
4014	}
4015
4016
4017	/**
4018	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4019	* areas. These MSRs will automatically be loaded to the host CPU on every
4020	* successful VM entry and stored from the host CPU on every successful VM exit.
4021	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4022	*
4023	* @returns VBox status code.
4024	* @param pVCpu Pointer to the VMCPU.
4025	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4026	* out-of-sync. Make sure to update the required fields
4027	* before using them.
4028	*
4029	* @remarks No-long-jump zone!!!
4030	*/
4031	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4032	{
4033	AssertPtr(pVCpu);
4034	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4035
4036	/*
4037	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
4038	*/
4039	int rc = VINF_SUCCESS;
4040	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4041	{
4042	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
4043	PVM pVM = pVCpu->CTX_SUFF(pVM);
4044	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4045	uint32_t cGuestMsrs = 0;
4046
4047	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
4048	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
4049	* when the guest really is in 64-bit mode. */
4050	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
4051	if (fSupportsLongMode)
4052	{
4053	pGuestMsr->u32Msr = MSR_K8_LSTAR;
4054	pGuestMsr->u32Reserved = 0;
4055	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
4056	pGuestMsr++; cGuestMsrs++;
4057	pGuestMsr->u32Msr = MSR_K6_STAR;
4058	pGuestMsr->u32Reserved = 0;
4059	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
4060	pGuestMsr++; cGuestMsrs++;
4061	pGuestMsr->u32Msr = MSR_K8_SF_MASK;
4062	pGuestMsr->u32Reserved = 0;
4063	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
4064	pGuestMsr++; cGuestMsrs++;
4065	pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
4066	pGuestMsr->u32Reserved = 0;
4067	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
4068	pGuestMsr++; cGuestMsrs++;
4069	}
4070
4071	/*
4072	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
4073	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
4074	*/
4075	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
4076	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
4077	{
4078	pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
4079	pGuestMsr->u32Reserved = 0;
4080	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
4081	AssertRCReturn(rc, rc);
4082	pGuestMsr++; cGuestMsrs++;
4083	}
4084
4085	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
4086	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
4087	{
4088	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
4089	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
4090	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4091	}
4092
4093	/* Update the VCPU's copy of the guest MSR count. */
4094	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
4095	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4096	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4097	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
4098
4099	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4100	}
4101
4102	/*
4103	* Guest Sysenter MSRs.
4104	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4105	* VM-exits on WRMSRs for these MSRs.
4106	*/
4107	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4108	{
4109	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4110	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4111	}
4112
4113	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4114	{
4115	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4116	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4117	}
4118
4119	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4120	{
4121	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4122	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4123	}
4124
4125	return rc;
4126	}
4127
4128
4129	/**
4130	* Loads the guest activity state into the guest-state area in the VMCS.
4131	*
4132	* @returns VBox status code.
4133	* @param pVCpu Pointer to the VMCPU.
4134	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4135	* out-of-sync. Make sure to update the required fields
4136	* before using them.
4137	*
4138	* @remarks No-long-jump zone!!!
4139	*/
4140	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4141	{
4142	/** @todo See if we can make use of other states, e.g.
4143	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4144	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4145	{
4146	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4147	AssertRCReturn(rc, rc);
4148
4149	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4150	}
4151	return VINF_SUCCESS;
4152	}
4153
4154
4155	/**
4156	* Sets up the appropriate function to run guest code.
4157	*
4158	* @returns VBox status code.
4159	* @param pVCpu Pointer to the VMCPU.
4160	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4161	* out-of-sync. Make sure to update the required fields
4162	* before using them.
4163	*
4164	* @remarks No-long-jump zone!!!
4165	*/
4166	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4167	{
4168	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4169	{
4170	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4171	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4172	#endif
4173	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4174	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4175	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4176	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4177	{
4178	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4179	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4180	}
4181	#else
4182	/* 64-bit host or hybrid host. */
4183	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4184	#endif
4185	}
4186	else
4187	{
4188	/* Guest is not in long mode, use the 32-bit handler. */
4189	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4190	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4191	{
4192	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4193	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4194	}
4195	#else
4196	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4197	#endif
4198	}
4199	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4200	return VINF_SUCCESS;
4201	}
4202
4203
4204	/**
4205	* Wrapper for running the guest code in VT-x.
4206	*
4207	* @returns VBox strict status code.
4208	* @param pVM Pointer to the VM.
4209	* @param pVCpu Pointer to the VMCPU.
4210	* @param pCtx Pointer to the guest-CPU context.
4211	*
4212	* @remarks No-long-jump zone!!!
4213	*/
4214	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4215	{
4216	/*
4217	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4218	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4219	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4220	*/
4221	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4222	/** @todo Add stats for resume vs launch. */
4223	#ifdef VBOX_WITH_KERNEL_USING_XMM
4224	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4225	#else
4226	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4227	#endif
4228	}
4229
4230
4231	/**
4232	* Reports world-switch error and dumps some useful debug info.
4233	*
4234	* @param pVM Pointer to the VM.
4235	* @param pVCpu Pointer to the VMCPU.
4236	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4237	* @param pCtx Pointer to the guest-CPU context.
4238	* @param pVmxTransient Pointer to the VMX transient structure (only
4239	* exitReason updated).
4240	*/
4241	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4242	{
4243	Assert(pVM);
4244	Assert(pVCpu);
4245	Assert(pCtx);
4246	Assert(pVmxTransient);
4247	HMVMX_ASSERT_PREEMPT_SAFE();
4248
4249	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4250	switch (rcVMRun)
4251	{
4252	case VERR_VMX_INVALID_VMXON_PTR:
4253	AssertFailed();
4254	break;
4255	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4256	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4257	{
4258	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4259	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4260	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4261	AssertRC(rc);
4262
4263	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4264	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4265	Cannot do it here as we may have been long preempted. */
4266
4267	#ifdef VBOX_STRICT
4268	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4269	pVmxTransient->uExitReason));
4270	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4271	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4272	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4273	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4274	else
4275	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4276	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4277	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4278
4279	/* VMX control bits. */
4280	uint32_t u32Val;
4281	uint64_t u64Val;
4282	HMVMXHCUINTREG uHCReg;
4283	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4284	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4285	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4286	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4287	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4288	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4289	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4290	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4291	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4292	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4293	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4294	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4295	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4296	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4297	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4298	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4299	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4300	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4301	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4302	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4303	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4304	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4305	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4306	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4307	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4308	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4309	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4310	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4311	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4312	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4313	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4314	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4315	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4316	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4317	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4318	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4319	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4320	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4321	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4322	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4323	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4324	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4325
4326	/* Guest bits. */
4327	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4328	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4329	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4330	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4331	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4332	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4333	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4334	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4335
4336	/* Host bits. */
4337	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4338	Log4(("Host CR0 %#RHr\n", uHCReg));
4339	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4340	Log4(("Host CR3 %#RHr\n", uHCReg));
4341	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4342	Log4(("Host CR4 %#RHr\n", uHCReg));
4343
4344	RTGDTR HostGdtr;
4345	PCX86DESCHC pDesc;
4346	ASMGetGDTR(&HostGdtr);
4347	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4348	Log4(("Host CS %#08x\n", u32Val));
4349	if (u32Val < HostGdtr.cbGdt)
4350	{
4351	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4352	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4353	}
4354
4355	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4356	Log4(("Host DS %#08x\n", u32Val));
4357	if (u32Val < HostGdtr.cbGdt)
4358	{
4359	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4360	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4361	}
4362
4363	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4364	Log4(("Host ES %#08x\n", u32Val));
4365	if (u32Val < HostGdtr.cbGdt)
4366	{
4367	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4368	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4369	}
4370
4371	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4372	Log4(("Host FS %#08x\n", u32Val));
4373	if (u32Val < HostGdtr.cbGdt)
4374	{
4375	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4376	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4377	}
4378
4379	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4380	Log4(("Host GS %#08x\n", u32Val));
4381	if (u32Val < HostGdtr.cbGdt)
4382	{
4383	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4384	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4385	}
4386
4387	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4388	Log4(("Host SS %#08x\n", u32Val));
4389	if (u32Val < HostGdtr.cbGdt)
4390	{
4391	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4392	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4393	}
4394
4395	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4396	Log4(("Host TR %#08x\n", u32Val));
4397	if (u32Val < HostGdtr.cbGdt)
4398	{
4399	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4400	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4401	}
4402
4403	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4404	Log4(("Host TR Base %#RHv\n", uHCReg));
4405	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4406	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4407	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4408	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4409	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4410	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4411	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4412	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4413	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4414	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4415	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4416	Log4(("Host RSP %#RHv\n", uHCReg));
4417	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4418	Log4(("Host RIP %#RHv\n", uHCReg));
4419	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4420	if (HMVMX_IS_64BIT_HOST_MODE())
4421	{
4422	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4423	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4424	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4425	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4426	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4427	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4428	}
4429	# endif
4430	#endif /* VBOX_STRICT */
4431	break;
4432	}
4433
4434	default:
4435	/* Impossible */
4436	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4437	break;
4438	}
4439	NOREF(pVM);
4440	}
4441
4442
4443	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4444	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4445	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4446	#endif
4447	#ifdef VBOX_STRICT
4448	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4449	{
4450	switch (idxField)
4451	{
4452	case VMX_VMCS_GUEST_RIP:
4453	case VMX_VMCS_GUEST_RSP:
4454	case VMX_VMCS_GUEST_SYSENTER_EIP:
4455	case VMX_VMCS_GUEST_SYSENTER_ESP:
4456	case VMX_VMCS_GUEST_GDTR_BASE:
4457	case VMX_VMCS_GUEST_IDTR_BASE:
4458	case VMX_VMCS_GUEST_CS_BASE:
4459	case VMX_VMCS_GUEST_DS_BASE:
4460	case VMX_VMCS_GUEST_ES_BASE:
4461	case VMX_VMCS_GUEST_FS_BASE:
4462	case VMX_VMCS_GUEST_GS_BASE:
4463	case VMX_VMCS_GUEST_SS_BASE:
4464	case VMX_VMCS_GUEST_LDTR_BASE:
4465	case VMX_VMCS_GUEST_TR_BASE:
4466	case VMX_VMCS_GUEST_CR3:
4467	return true;
4468	}
4469	return false;
4470	}
4471
4472	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4473	{
4474	switch (idxField)
4475	{
4476	/* Read-only fields. */
4477	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4478	return true;
4479	}
4480	/* Remaining readable fields should also be writable. */
4481	return hmR0VmxIsValidWriteField(idxField);
4482	}
4483	#endif /* VBOX_STRICT */
4484
4485
4486	/**
4487	* Executes the specified handler in 64-bit mode.
4488	*
4489	* @returns VBox status code.
4490	* @param pVM Pointer to the VM.
4491	* @param pVCpu Pointer to the VMCPU.
4492	* @param pCtx Pointer to the guest CPU context.
4493	* @param enmOp The operation to perform.
4494	* @param cbParam Number of parameters.
4495	* @param paParam Array of 32-bit parameters.
4496	*/
4497	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4498	uint32_t *paParam)
4499	{
4500	int rc, rc2;
4501	PHMGLOBALCPUINFO pCpu;
4502	RTHCPHYS HCPhysCpuPage;
4503	RTCCUINTREG uOldEflags;
4504
4505	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4506	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4507	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4508	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4509
4510	#ifdef VBOX_STRICT
4511	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4512	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4513
4514	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4515	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4516	#endif
4517
4518	/* Disable interrupts. */
4519	uOldEflags = ASMIntDisableFlags();
4520
4521	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4522	RTCPUID idHostCpu = RTMpCpuId();
4523	CPUMR0SetLApic(pVCpu, idHostCpu);
4524	#endif
4525
4526	pCpu = HMR0GetCurrentCpu();
4527	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4528
4529	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4530	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4531
4532	/* Leave VMX Root Mode. */
4533	VMXDisable();
4534
4535	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4536
4537	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4538	CPUMSetHyperEIP(pVCpu, enmOp);
4539	for (int i = (int)cbParam - 1; i >= 0; i--)
4540	CPUMPushHyper(pVCpu, paParam[i]);
4541
4542	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4543
4544	/* Call the switcher. */
4545	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4546	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4547
4548	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4549	/* Make sure the VMX instructions don't cause #UD faults. */
4550	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4551
4552	/* Re-enter VMX Root Mode */
4553	rc2 = VMXEnable(HCPhysCpuPage);
4554	if (RT_FAILURE(rc2))
4555	{
4556	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4557	ASMSetFlags(uOldEflags);
4558	return rc2;
4559	}
4560
4561	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4562	AssertRC(rc2);
4563	Assert(!(ASMGetFlags() & X86_EFL_IF));
4564	ASMSetFlags(uOldEflags);
4565	return rc;
4566	}
4567
4568
4569	/**
4570	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4571	* supporting 64-bit guests.
4572	*
4573	* @returns VBox status code.
4574	* @param fResume Whether to VMLAUNCH or VMRESUME.
4575	* @param pCtx Pointer to the guest-CPU context.
4576	* @param pCache Pointer to the VMCS cache.
4577	* @param pVM Pointer to the VM.
4578	* @param pVCpu Pointer to the VMCPU.
4579	*/
4580	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4581	{
4582	uint32_t aParam[6];
4583	PHMGLOBALCPUINFO pCpu = NULL;
4584	RTHCPHYS HCPhysCpuPage = 0;
4585	int rc = VERR_INTERNAL_ERROR_5;
4586
4587	pCpu = HMR0GetCurrentCpu();
4588	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4589
4590	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4591	pCache->uPos = 1;
4592	pCache->interPD = PGMGetInterPaeCR3(pVM);
4593	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4594	#endif
4595
4596	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4597	pCache->TestIn.HCPhysCpuPage = 0;
4598	pCache->TestIn.HCPhysVmcs = 0;
4599	pCache->TestIn.pCache = 0;
4600	pCache->TestOut.HCPhysVmcs = 0;
4601	pCache->TestOut.pCache = 0;
4602	pCache->TestOut.pCtx = 0;
4603	pCache->TestOut.eflags = 0;
4604	#endif
4605
4606	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4607	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4608	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4609	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4610	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4611	aParam[5] = 0;
4612
4613	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4614	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4615	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4616	#endif
4617	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4618
4619	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4620	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4621	Assert(pCtx->dr[4] == 10);
4622	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4623	#endif
4624
4625	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4626	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4627	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4628	pVCpu->hm.s.vmx.HCPhysVmcs));
4629	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4630	pCache->TestOut.HCPhysVmcs));
4631	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4632	pCache->TestOut.pCache));
4633	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4634	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4635	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4636	pCache->TestOut.pCtx));
4637	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4638	#endif
4639	return rc;
4640	}
4641
4642
4643	/**
4644	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4645	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4646	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4647	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4648	*
4649	* @returns VBox status code.
4650	* @param pVM Pointer to the VM.
4651	* @param pVCpu Pointer to the VMCPU.
4652	*/
4653	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4654	{
4655	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4656	{ \
4657	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4658	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4659	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4660	++cReadFields; \
4661	}
4662
4663	AssertPtr(pVM);
4664	AssertPtr(pVCpu);
4665	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4666	uint32_t cReadFields = 0;
4667
4668	/*
4669	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4670	* and serve to indicate exceptions to the rules.
4671	*/
4672
4673	/* Guest-natural selector base fields. */
4674	#if 0
4675	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4676	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4677	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4678	#endif
4679	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4680	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4681	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4682	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4683	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4684	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4685	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4686	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4687	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4688	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4689	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4690	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4691	#if 0
4692	/* Unused natural width guest-state fields. */
4693	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4694	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4695	#endif
4696	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4697	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4698
4699	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4700	#if 0
4701	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4702	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4703	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4704	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4705	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4706	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4707	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4708	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4709	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4710	#endif
4711
4712	/* Natural width guest-state fields. */
4713	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4714	#if 0
4715	/* Currently unused field. */
4716	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4717	#endif
4718
4719	if (pVM->hm.s.fNestedPaging)
4720	{
4721	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4722	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4723	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4724	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4725	}
4726	else
4727	{
4728	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4729	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4730	}
4731
4732	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4733	return VINF_SUCCESS;
4734	}
4735
4736
4737	/**
4738	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4739	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4740	* darwin, running 64-bit guests).
4741	*
4742	* @returns VBox status code.
4743	* @param pVCpu Pointer to the VMCPU.
4744	* @param idxField The VMCS field encoding.
4745	* @param u64Val 16, 32 or 64 bits value.
4746	*/
4747	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4748	{
4749	int rc;
4750	switch (idxField)
4751	{
4752	/*
4753	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4754	*/
4755	/* 64-bit Control fields. */
4756	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4757	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4758	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4759	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4760	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4761	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4762	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4763	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4764	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4765	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4766	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4767	case VMX_VMCS64_CTRL_EPTP_FULL:
4768	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4769	/* 64-bit Guest-state fields. */
4770	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4771	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4772	case VMX_VMCS64_GUEST_PAT_FULL:
4773	case VMX_VMCS64_GUEST_EFER_FULL:
4774	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4775	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4776	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4777	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4778	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4779	/* 64-bit Host-state fields. */
4780	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4781	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4782	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4783	{
4784	rc = VMXWriteVmcs32(idxField, u64Val);
4785	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4786	break;
4787	}
4788
4789	/*
4790	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4791	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4792	*/
4793	/* Natural-width Guest-state fields. */
4794	case VMX_VMCS_GUEST_CR3:
4795	case VMX_VMCS_GUEST_ES_BASE:
4796	case VMX_VMCS_GUEST_CS_BASE:
4797	case VMX_VMCS_GUEST_SS_BASE:
4798	case VMX_VMCS_GUEST_DS_BASE:
4799	case VMX_VMCS_GUEST_FS_BASE:
4800	case VMX_VMCS_GUEST_GS_BASE:
4801	case VMX_VMCS_GUEST_LDTR_BASE:
4802	case VMX_VMCS_GUEST_TR_BASE:
4803	case VMX_VMCS_GUEST_GDTR_BASE:
4804	case VMX_VMCS_GUEST_IDTR_BASE:
4805	case VMX_VMCS_GUEST_RSP:
4806	case VMX_VMCS_GUEST_RIP:
4807	case VMX_VMCS_GUEST_SYSENTER_ESP:
4808	case VMX_VMCS_GUEST_SYSENTER_EIP:
4809	{
4810	if (!(u64Val >> 32))
4811	{
4812	/* If this field is 64-bit, VT-x will zero out the top bits. */
4813	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4814	}
4815	else
4816	{
4817	/* Assert that only the 32->64 switcher case should ever come here. */
4818	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4819	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4820	}
4821	break;
4822	}
4823
4824	default:
4825	{
4826	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4827	rc = VERR_INVALID_PARAMETER;
4828	break;
4829	}
4830	}
4831	AssertRCReturn(rc, rc);
4832	return rc;
4833	}
4834
4835
4836	/**
4837	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4838	* hosts (except darwin) for 64-bit guests.
4839	*
4840	* @param pVCpu Pointer to the VMCPU.
4841	* @param idxField The VMCS field encoding.
4842	* @param u64Val 16, 32 or 64 bits value.
4843	*/
4844	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4845	{
4846	AssertPtr(pVCpu);
4847	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4848
4849	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4850	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4851
4852	/* Make sure there are no duplicates. */
4853	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4854	{
4855	if (pCache->Write.aField[i] == idxField)
4856	{
4857	pCache->Write.aFieldVal[i] = u64Val;
4858	return VINF_SUCCESS;
4859	}
4860	}
4861
4862	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4863	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4864	pCache->Write.cValidEntries++;
4865	return VINF_SUCCESS;
4866	}
4867
4868	/* Enable later when the assembly code uses these as callbacks. */
4869	#if 0
4870	/*
4871	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4872	*
4873	* @param pVCpu Pointer to the VMCPU.
4874	* @param pCache Pointer to the VMCS cache.
4875	*
4876	* @remarks No-long-jump zone!!!
4877	*/
4878	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4879	{
4880	AssertPtr(pCache);
4881	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4882	{
4883	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4884	AssertRC(rc);
4885	}
4886	pCache->Write.cValidEntries = 0;
4887	}
4888
4889
4890	/**
4891	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4892	*
4893	* @param pVCpu Pointer to the VMCPU.
4894	* @param pCache Pointer to the VMCS cache.
4895	*
4896	* @remarks No-long-jump zone!!!
4897	*/
4898	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4899	{
4900	AssertPtr(pCache);
4901	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4902	{
4903	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4904	AssertRC(rc);
4905	}
4906	}
4907	#endif
4908	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4909
4910
4911	/**
4912	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4913	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4914	* timer.
4915	*
4916	* @returns VBox status code.
4917	* @param pVCpu Pointer to the VMCPU.
4918	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4919	* out-of-sync. Make sure to update the required fields
4920	* before using them.
4921	* @remarks No-long-jump zone!!!
4922	*/
4923	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4924	{
4925	int rc = VERR_INTERNAL_ERROR_5;
4926	bool fOffsettedTsc = false;
4927	PVM pVM = pVCpu->CTX_SUFF(pVM);
4928	if (pVM->hm.s.vmx.fUsePreemptTimer)
4929	{
4930	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4931
4932	/* Make sure the returned values have sane upper and lower boundaries. */
4933	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4934	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4935	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4936	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4937
4938	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4939	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4940	}
4941	else
4942	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4943
4944	if (fOffsettedTsc)
4945	{
4946	uint64_t u64CurTSC = ASMReadTSC();
4947	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4948	{
4949	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4950	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4951
4952	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4953	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4954	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4955	}
4956	else
4957	{
4958	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4959	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4960	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4961	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4962	}
4963	}
4964	else
4965	{
4966	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4967	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4968	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4969	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4970	}
4971	}
4972
4973
4974	/**
4975	* Determines if an exception is a contributory exception. Contributory
4976	* exceptions are ones which can cause double-faults. Page-fault is
4977	* intentionally not included here as it's a conditional contributory exception.
4978	*
4979	* @returns true if the exception is contributory, false otherwise.
4980	* @param uVector The exception vector.
4981	*/
4982	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4983	{
4984	switch (uVector)
4985	{
4986	case X86_XCPT_GP:
4987	case X86_XCPT_SS:
4988	case X86_XCPT_NP:
4989	case X86_XCPT_TS:
4990	case X86_XCPT_DE:
4991	return true;
4992	default:
4993	break;
4994	}
4995	return false;
4996	}
4997
4998
4999	/**
5000	* Sets an event as a pending event to be injected into the guest.
5001	*
5002	* @param pVCpu Pointer to the VMCPU.
5003	* @param u32IntInfo The VM-entry interruption-information field.
5004	* @param cbInstr The VM-entry instruction length in bytes (for software
5005	* interrupts, exceptions and privileged software
5006	* exceptions).
5007	* @param u32ErrCode The VM-entry exception error code.
5008	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5009	* page-fault.
5010	*
5011	* @remarks Statistics counter assumes this is a guest event being injected or
5012	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5013	* always incremented.
5014	*/
5015	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5016	RTGCUINTPTR GCPtrFaultAddress)
5017	{
5018	Assert(!pVCpu->hm.s.Event.fPending);
5019	pVCpu->hm.s.Event.fPending = true;
5020	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5021	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5022	pVCpu->hm.s.Event.cbInstr = cbInstr;
5023	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5024
5025	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5026	}
5027
5028
5029	/**
5030	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5031	*
5032	* @param pVCpu Pointer to the VMCPU.
5033	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5034	* out-of-sync. Make sure to update the required fields
5035	* before using them.
5036	*/
5037	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5038	{
5039	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5040	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5041	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5042	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5043	}
5044
5045
5046	/**
5047	* Handle a condition that occurred while delivering an event through the guest
5048	* IDT.
5049	*
5050	* @returns VBox status code (informational error codes included).
5051	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5052	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5053	* continue execution of the guest which will delivery the #DF.
5054	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5055	*
5056	* @param pVCpu Pointer to the VMCPU.
5057	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5058	* out-of-sync. Make sure to update the required fields
5059	* before using them.
5060	* @param pVmxTransient Pointer to the VMX transient structure.
5061	*
5062	* @remarks No-long-jump zone!!!
5063	*/
5064	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5065	{
5066	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5067	AssertRCReturn(rc, rc);
5068	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5069	{
5070	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
5071	AssertRCReturn(rc, rc);
5072
5073	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5074	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5075	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5076
5077	typedef enum
5078	{
5079	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5080	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5081	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5082	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5083	} VMXREFLECTXCPT;
5084
5085	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5086	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5087	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5088	{
5089	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5090	{
5091	enmReflect = VMXREFLECTXCPT_XCPT;
5092	#ifdef VBOX_STRICT
5093	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5094	&& uExitVector == X86_XCPT_PF)
5095	{
5096	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5097	}
5098	#endif
5099	if ( uExitVector == X86_XCPT_PF
5100	&& uIdtVector == X86_XCPT_PF)
5101	{
5102	pVmxTransient->fVectoringPF = true;
5103	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5104	}
5105	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5106	&& hmR0VmxIsContributoryXcpt(uExitVector)
5107	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5108	\|\| uIdtVector == X86_XCPT_PF))
5109	{
5110	enmReflect = VMXREFLECTXCPT_DF;
5111	}
5112	else if (uIdtVector == X86_XCPT_DF)
5113	enmReflect = VMXREFLECTXCPT_TF;
5114	}
5115	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5116	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5117	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5118	{
5119	/*
5120	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5121	* (whatever they are) as they reoccur when restarting the instruction.
5122	*/
5123	enmReflect = VMXREFLECTXCPT_XCPT;
5124	}
5125	}
5126	else
5127	{
5128	/*
5129	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5130	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5131	* original exception to the guest after handling the VM-exit.
5132	*/
5133	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5134	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5135	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5136	{
5137	enmReflect = VMXREFLECTXCPT_XCPT;
5138	}
5139	}
5140
5141	switch (enmReflect)
5142	{
5143	case VMXREFLECTXCPT_XCPT:
5144	{
5145	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5146	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5147	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5148
5149	uint32_t u32ErrCode = 0;
5150	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5151	{
5152	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5153	AssertRCReturn(rc, rc);
5154	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5155	}
5156
5157	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5158	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5159	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5160	rc = VINF_SUCCESS;
5161	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5162	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5163
5164	break;
5165	}
5166
5167	case VMXREFLECTXCPT_DF:
5168	{
5169	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5170	rc = VINF_HM_DOUBLE_FAULT;
5171	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5172	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5173
5174	break;
5175	}
5176
5177	case VMXREFLECTXCPT_TF:
5178	{
5179	rc = VINF_EM_RESET;
5180	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5181	uExitVector));
5182	break;
5183	}
5184
5185	default:
5186	Assert(rc == VINF_SUCCESS);
5187	break;
5188	}
5189	}
5190	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5191	return rc;
5192	}
5193
5194
5195	/**
5196	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5197	*
5198	* @returns VBox status code.
5199	* @param pVCpu Pointer to the VMCPU.
5200	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5201	* out-of-sync. Make sure to update the required fields
5202	* before using them.
5203	*
5204	* @remarks No-long-jump zone!!!
5205	*/
5206	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5207	{
5208	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5209	{
5210	uint32_t uVal = 0;
5211	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5212	AssertRCReturn(rc, rc);
5213
5214	uint32_t uShadow = 0;
5215	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5216	AssertRCReturn(rc, rc);
5217
5218	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5219	CPUMSetGuestCR0(pVCpu, uVal);
5220	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5221	}
5222	return VINF_SUCCESS;
5223	}
5224
5225
5226	/**
5227	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5228	*
5229	* @returns VBox status code.
5230	* @param pVCpu Pointer to the VMCPU.
5231	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5232	* out-of-sync. Make sure to update the required fields
5233	* before using them.
5234	*
5235	* @remarks No-long-jump zone!!!
5236	*/
5237	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5238	{
5239	int rc = VINF_SUCCESS;
5240	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5241	{
5242	uint32_t uVal = 0;
5243	uint32_t uShadow = 0;
5244	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5245	AssertRCReturn(rc, rc);
5246	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5247	AssertRCReturn(rc, rc);
5248
5249	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5250	CPUMSetGuestCR4(pVCpu, uVal);
5251	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5252	}
5253	return rc;
5254	}
5255
5256
5257	/**
5258	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5259	*
5260	* @returns VBox status code.
5261	* @param pVCpu Pointer to the VMCPU.
5262	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5263	* out-of-sync. Make sure to update the required fields
5264	* before using them.
5265	*
5266	* @remarks No-long-jump zone!!!
5267	*/
5268	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5269	{
5270	int rc = VINF_SUCCESS;
5271	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5272	{
5273	uint64_t u64Val = 0;
5274	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5275	AssertRCReturn(rc, rc);
5276
5277	pMixedCtx->rip = u64Val;
5278	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5279	}
5280	return rc;
5281	}
5282
5283
5284	/**
5285	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5286	*
5287	* @returns VBox status code.
5288	* @param pVCpu Pointer to the VMCPU.
5289	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5290	* out-of-sync. Make sure to update the required fields
5291	* before using them.
5292	*
5293	* @remarks No-long-jump zone!!!
5294	*/
5295	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5296	{
5297	int rc = VINF_SUCCESS;
5298	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5299	{
5300	uint64_t u64Val = 0;
5301	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5302	AssertRCReturn(rc, rc);
5303
5304	pMixedCtx->rsp = u64Val;
5305	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5306	}
5307	return rc;
5308	}
5309
5310
5311	/**
5312	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5313	*
5314	* @returns VBox status code.
5315	* @param pVCpu Pointer to the VMCPU.
5316	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5317	* out-of-sync. Make sure to update the required fields
5318	* before using them.
5319	*
5320	* @remarks No-long-jump zone!!!
5321	*/
5322	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5323	{
5324	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5325	{
5326	uint32_t uVal = 0;
5327	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5328	AssertRCReturn(rc, rc);
5329
5330	pMixedCtx->eflags.u32 = uVal;
5331	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5332	{
5333	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5334	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5335
5336	pMixedCtx->eflags.Bits.u1VM = 0;
5337	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5338	}
5339
5340	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5341	}
5342	return VINF_SUCCESS;
5343	}
5344
5345
5346	/**
5347	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5348	* guest-CPU context.
5349	*/
5350	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5351	{
5352	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5353	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5354	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5355	return rc;
5356	}
5357
5358
5359	/**
5360	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5361	* from the guest-state area in the VMCS.
5362	*
5363	* @param pVCpu Pointer to the VMCPU.
5364	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5365	* out-of-sync. Make sure to update the required fields
5366	* before using them.
5367	*
5368	* @remarks No-long-jump zone!!!
5369	*/
5370	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5371	{
5372	uint32_t uIntrState = 0;
5373	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5374	AssertRC(rc);
5375
5376	if (!uIntrState)
5377	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5378	else
5379	{
5380	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5381	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5382	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5383	AssertRC(rc);
5384	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5385	AssertRC(rc);
5386
5387	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5388	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5389	}
5390	}
5391
5392
5393	/**
5394	* Saves the guest's activity state.
5395	*
5396	* @returns VBox status code.
5397	* @param pVCpu Pointer to the VMCPU.
5398	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5399	* out-of-sync. Make sure to update the required fields
5400	* before using them.
5401	*
5402	* @remarks No-long-jump zone!!!
5403	*/
5404	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5405	{
5406	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5407	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5408	return VINF_SUCCESS;
5409	}
5410
5411
5412	/**
5413	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5414	* the current VMCS into the guest-CPU context.
5415	*
5416	* @returns VBox status code.
5417	* @param pVCpu Pointer to the VMCPU.
5418	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5419	* out-of-sync. Make sure to update the required fields
5420	* before using them.
5421	*
5422	* @remarks No-long-jump zone!!!
5423	*/
5424	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5425	{
5426	int rc = VINF_SUCCESS;
5427	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5428	{
5429	uint32_t u32Val = 0;
5430	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5431	pMixedCtx->SysEnter.cs = u32Val;
5432	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5433	}
5434
5435	uint64_t u64Val = 0;
5436	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5437	{
5438	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5439	pMixedCtx->SysEnter.eip = u64Val;
5440	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5441	}
5442	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5443	{
5444	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5445	pMixedCtx->SysEnter.esp = u64Val;
5446	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5447	}
5448	return rc;
5449	}
5450
5451
5452	/**
5453	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5454	* context.
5455	*
5456	* @returns VBox status code.
5457	* @param pVCpu Pointer to the VMCPU.
5458	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5459	* out-of-sync. Make sure to update the required fields
5460	* before using them.
5461	*
5462	* @remarks No-long-jump zone!!!
5463	*/
5464	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5465	{
5466	int rc = VINF_SUCCESS;
5467	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5468	{
5469	uint64_t u64Val = 0;
5470	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5471	pMixedCtx->fs.u64Base = u64Val;
5472	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5473	}
5474	return rc;
5475	}
5476
5477
5478	/**
5479	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5480	* context.
5481	*
5482	* @returns VBox status code.
5483	* @param pVCpu Pointer to the VMCPU.
5484	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5485	* out-of-sync. Make sure to update the required fields
5486	* before using them.
5487	*
5488	* @remarks No-long-jump zone!!!
5489	*/
5490	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5491	{
5492	int rc = VINF_SUCCESS;
5493	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5494	{
5495	uint64_t u64Val = 0;
5496	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5497	pMixedCtx->gs.u64Base = u64Val;
5498	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5499	}
5500	return rc;
5501	}
5502
5503
5504	/**
5505	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5506	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5507	* and TSC_AUX.
5508	*
5509	* @returns VBox status code.
5510	* @param pVCpu Pointer to the VMCPU.
5511	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5512	* out-of-sync. Make sure to update the required fields
5513	* before using them.
5514	*
5515	* @remarks No-long-jump zone!!!
5516	*/
5517	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5518	{
5519	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5520	return VINF_SUCCESS;
5521
5522	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5523	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5524	{
5525	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5526	pMsr += i;
5527	switch (pMsr->u32Msr)
5528	{
5529	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5530	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5531	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5532	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5533	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5534	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5535	default:
5536	{
5537	AssertFailed();
5538	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5539	}
5540	}
5541	}
5542	#endif
5543
5544	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5545	return VINF_SUCCESS;
5546	}
5547
5548
5549	/**
5550	* Saves the guest control registers from the current VMCS into the guest-CPU
5551	* context.
5552	*
5553	* @returns VBox status code.
5554	* @param pVCpu Pointer to the VMCPU.
5555	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5556	* out-of-sync. Make sure to update the required fields
5557	* before using them.
5558	*
5559	* @remarks No-long-jump zone!!!
5560	*/
5561	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5562	{
5563	/* Guest CR0. Guest FPU. */
5564	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5565	AssertRCReturn(rc, rc);
5566
5567	/* Guest CR4. */
5568	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5569	AssertRCReturn(rc, rc);
5570
5571	/* Guest CR2 - updated always during the world-switch or in #PF. */
5572	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5573	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5574	{
5575	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5576	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5577
5578	PVM pVM = pVCpu->CTX_SUFF(pVM);
5579	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5580	\|\| ( pVM->hm.s.fNestedPaging
5581	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5582	{
5583	uint64_t u64Val = 0;
5584	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5585	if (pMixedCtx->cr3 != u64Val)
5586	{
5587	CPUMSetGuestCR3(pVCpu, u64Val);
5588	if (VMMRZCallRing3IsEnabled(pVCpu))
5589	{
5590	PGMUpdateCR3(pVCpu, u64Val);
5591	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5592	}
5593	else
5594	{
5595	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5596	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5597	}
5598	}
5599
5600	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5601	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5602	{
5603	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5604	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5605	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5606	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5607
5608	if (VMMRZCallRing3IsEnabled(pVCpu))
5609	{
5610	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5611	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5612	}
5613	else
5614	{
5615	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5616	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5617	}
5618	}
5619	}
5620
5621	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5622	}
5623
5624	/*
5625	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5626	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5627	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5628	*
5629	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5630	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5631	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5632	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
5633	*
5634	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5635	*/
5636	if (VMMRZCallRing3IsEnabled(pVCpu))
5637	{
5638	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5639	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5640
5641	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5642	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5643
5644	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5645	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5646	}
5647
5648	return rc;
5649	}
5650
5651
5652	/**
5653	* Reads a guest segment register from the current VMCS into the guest-CPU
5654	* context.
5655	*
5656	* @returns VBox status code.
5657	* @param pVCpu Pointer to the VMCPU.
5658	* @param idxSel Index of the selector in the VMCS.
5659	* @param idxLimit Index of the segment limit in the VMCS.
5660	* @param idxBase Index of the segment base in the VMCS.
5661	* @param idxAccess Index of the access rights of the segment in the VMCS.
5662	* @param pSelReg Pointer to the segment selector.
5663	*
5664	* @remarks No-long-jump zone!!!
5665	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5666	* macro as that takes care of whether to read from the VMCS cache or
5667	* not.
5668	*/
5669	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5670	PCPUMSELREG pSelReg)
5671	{
5672	uint32_t u32Val = 0;
5673	int rc = VMXReadVmcs32(idxSel, &u32Val);
5674	AssertRCReturn(rc, rc);
5675	pSelReg->Sel = (uint16_t)u32Val;
5676	pSelReg->ValidSel = (uint16_t)u32Val;
5677	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5678
5679	rc = VMXReadVmcs32(idxLimit, &u32Val);
5680	AssertRCReturn(rc, rc);
5681	pSelReg->u32Limit = u32Val;
5682
5683	uint64_t u64Val = 0;
5684	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5685	AssertRCReturn(rc, rc);
5686	pSelReg->u64Base = u64Val;
5687
5688	rc = VMXReadVmcs32(idxAccess, &u32Val);
5689	AssertRCReturn(rc, rc);
5690	pSelReg->Attr.u = u32Val;
5691
5692	/*
5693	* If VT-x marks the segment as unusable, most other bits remain undefined:
5694	* - For CS the L, D and G bits have meaning.
5695	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5696	* - For the remaining data segments no bits are defined.
5697	*
5698	* The present bit and the unusable bit has been observed to be set at the
5699	* same time (the selector was supposed to invalid as we started executing
5700	* a V8086 interrupt in ring-0).
5701	*
5702	* What should be important for the rest of the VBox code that the P bit is
5703	* cleared. Some of the other VBox code recognizes the unusable bit, but
5704	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
5705	* safe side here, we'll strip off P and other bits we don't care about. If
5706	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5707	*
5708	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5709	*/
5710	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5711	{
5712	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5713
5714	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5715	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
5716	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
5717
5718	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5719	#ifdef DEBUG_bird
5720	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5721	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5722	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5723	#endif
5724	}
5725	return VINF_SUCCESS;
5726	}
5727
5728
5729	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5730	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5731	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5732	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5733	#else
5734	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5735	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5736	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5737	#endif
5738
5739
5740	/**
5741	* Saves the guest segment registers from the current VMCS into the guest-CPU
5742	* context.
5743	*
5744	* @returns VBox status code.
5745	* @param pVCpu Pointer to the VMCPU.
5746	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5747	* out-of-sync. Make sure to update the required fields
5748	* before using them.
5749	*
5750	* @remarks No-long-jump zone!!!
5751	*/
5752	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5753	{
5754	/* Guest segment registers. */
5755	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5756	{
5757	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5758	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5759	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5760	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5761	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5762	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5763	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5764
5765	/* Restore segment attributes for real-on-v86 mode hack. */
5766	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5767	{
5768	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5769	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5770	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5771	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5772	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5773	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5774	}
5775	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5776	}
5777
5778	return VINF_SUCCESS;
5779	}
5780
5781
5782	/**
5783	* Saves the guest descriptor table registers and task register from the current
5784	* VMCS into the guest-CPU context.
5785	*
5786	* @returns VBox status code.
5787	* @param pVCpu Pointer to the VMCPU.
5788	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5789	* out-of-sync. Make sure to update the required fields
5790	* before using them.
5791	*
5792	* @remarks No-long-jump zone!!!
5793	*/
5794	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5795	{
5796	int rc = VINF_SUCCESS;
5797
5798	/* Guest LDTR. */
5799	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5800	{
5801	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5802	AssertRCReturn(rc, rc);
5803	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5804	}
5805
5806	/* Guest GDTR. */
5807	uint64_t u64Val = 0;
5808	uint32_t u32Val = 0;
5809	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5810	{
5811	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5812	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5813	pMixedCtx->gdtr.pGdt = u64Val;
5814	pMixedCtx->gdtr.cbGdt = u32Val;
5815	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5816	}
5817
5818	/* Guest IDTR. */
5819	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5820	{
5821	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5822	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5823	pMixedCtx->idtr.pIdt = u64Val;
5824	pMixedCtx->idtr.cbIdt = u32Val;
5825	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5826	}
5827
5828	/* Guest TR. */
5829	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5830	{
5831	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5832	AssertRCReturn(rc, rc);
5833
5834	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5835	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5836	{
5837	rc = VMXLOCAL_READ_SEG(TR, tr);
5838	AssertRCReturn(rc, rc);
5839	}
5840	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5841	}
5842	return rc;
5843	}
5844
5845	#undef VMXLOCAL_READ_SEG
5846
5847
5848	/**
5849	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5850	* context.
5851	*
5852	* @returns VBox status code.
5853	* @param pVCpu Pointer to the VMCPU.
5854	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5855	* out-of-sync. Make sure to update the required fields
5856	* before using them.
5857	*
5858	* @remarks No-long-jump zone!!!
5859	*/
5860	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5861	{
5862	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5863	{
5864	if (!pVCpu->hm.s.fUsingHyperDR7)
5865	{
5866	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5867	uint32_t u32Val;
5868	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5869	pMixedCtx->dr[7] = u32Val;
5870	}
5871
5872	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5873	}
5874	return VINF_SUCCESS;
5875	}
5876
5877
5878	/**
5879	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
5880	*
5881	* @returns VBox status code.
5882	* @param pVCpu Pointer to the VMCPU.
5883	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5884	* out-of-sync. Make sure to update the required fields
5885	* before using them.
5886	*
5887	* @remarks No-long-jump zone!!!
5888	*/
5889	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5890	{
5891	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5892	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5893	return VINF_SUCCESS;
5894	}
5895
5896
5897	/**
5898	* Saves the entire guest state from the currently active VMCS into the
5899	* guest-CPU context. This essentially VMREADs all guest-data.
5900	*
5901	* @returns VBox status code.
5902	* @param pVCpu Pointer to the VMCPU.
5903	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5904	* out-of-sync. Make sure to update the required fields
5905	* before using them.
5906	*/
5907	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5908	{
5909	Assert(pVCpu);
5910	Assert(pMixedCtx);
5911
5912	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5913	return VINF_SUCCESS;
5914
5915	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
5916	again on the ring-3 callback path, there is no real need to. */
5917	if (VMMRZCallRing3IsEnabled(pVCpu))
5918	VMMR0LogFlushDisable(pVCpu);
5919	else
5920	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5921	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5922
5923	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5924	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5925
5926	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5927	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5928
5929	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5930	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5931
5932	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5933	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5934
5935	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5936	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5937
5938	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5939	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5940
5941	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5942	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5943
5944	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5945	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5946
5947	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5948	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5949
5950	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5951	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5952
5953	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5954	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5955
5956	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5957	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5958
5959	if (VMMRZCallRing3IsEnabled(pVCpu))
5960	VMMR0LogFlushEnable(pVCpu);
5961
5962	return rc;
5963	}
5964
5965
5966	/**
5967	* Check per-VM and per-VCPU force flag actions that require us to go back to
5968	* ring-3 for one reason or another.
5969	*
5970	* @returns VBox status code (information status code included).
5971	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5972	* ring-3.
5973	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5974	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5975	* interrupts)
5976	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5977	* all EMTs to be in ring-3.
5978	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5979	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5980	* to the EM loop.
5981	*
5982	* @param pVM Pointer to the VM.
5983	* @param pVCpu Pointer to the VMCPU.
5984	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5985	* out-of-sync. Make sure to update the required fields
5986	* before using them.
5987	*/
5988	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5989	{
5990	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5991
5992	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5993	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5994	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5995	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5996	{
5997	/* We need the control registers now, make sure the guest-CPU context is updated. */
5998	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5999	AssertRCReturn(rc3, rc3);
6000
6001	/* Pending HM CR3 sync. */
6002	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6003	{
6004	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6005	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6006	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6007	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6008	}
6009
6010	/* Pending HM PAE PDPEs. */
6011	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6012	{
6013	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6014	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6015	}
6016
6017	/* Pending PGM C3 sync. */
6018	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6019	{
6020	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6021	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6022	if (rc2 != VINF_SUCCESS)
6023	{
6024	AssertRC(rc2);
6025	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6026	return rc2;
6027	}
6028	}
6029
6030	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6031	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6032	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6033	{
6034	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6035	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6036	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6037	return rc2;
6038	}
6039
6040	/* Pending VM request packets, such as hardware interrupts. */
6041	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6042	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6043	{
6044	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6045	return VINF_EM_PENDING_REQUEST;
6046	}
6047
6048	/* Pending PGM pool flushes. */
6049	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6050	{
6051	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6052	return VINF_PGM_POOL_FLUSH_PENDING;
6053	}
6054
6055	/* Pending DMA requests. */
6056	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6057	{
6058	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6059	return VINF_EM_RAW_TO_R3;
6060	}
6061	}
6062
6063	return VINF_SUCCESS;
6064	}
6065
6066
6067	/**
6068	* Converts any TRPM trap into a pending HM event. This is typically used when
6069	* entering from ring-3 (not longjmp returns).
6070	*
6071	* @param pVCpu Pointer to the VMCPU.
6072	*/
6073	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6074	{
6075	Assert(TRPMHasTrap(pVCpu));
6076	Assert(!pVCpu->hm.s.Event.fPending);
6077
6078	uint8_t uVector;
6079	TRPMEVENT enmTrpmEvent;
6080	RTGCUINT uErrCode;
6081	RTGCUINTPTR GCPtrFaultAddress;
6082	uint8_t cbInstr;
6083
6084	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6085	AssertRC(rc);
6086
6087	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6088	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6089	if (enmTrpmEvent == TRPM_TRAP)
6090	{
6091	switch (uVector)
6092	{
6093	case X86_XCPT_BP:
6094	case X86_XCPT_OF:
6095	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6096	break;
6097
6098	case X86_XCPT_PF:
6099	case X86_XCPT_DF:
6100	case X86_XCPT_TS:
6101	case X86_XCPT_NP:
6102	case X86_XCPT_SS:
6103	case X86_XCPT_GP:
6104	case X86_XCPT_AC:
6105	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6106	/* no break! */
6107	default:
6108	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6109	break;
6110	}
6111	}
6112	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6113	{
6114	if (uVector == X86_XCPT_NMI)
6115	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6116	else
6117	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6118	}
6119	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6120	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6121	else
6122	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6123
6124	rc = TRPMResetTrap(pVCpu);
6125	AssertRC(rc);
6126	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6127	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6128
6129	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6130	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6131	}
6132
6133
6134	/**
6135	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6136	* VT-x to execute any instruction.
6137	*
6138	* @param pvCpu Pointer to the VMCPU.
6139	*/
6140	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6141	{
6142	Assert(pVCpu->hm.s.Event.fPending);
6143
6144	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6145	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6146	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6147	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6148
6149	/* If a trap was already pending, we did something wrong! */
6150	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6151
6152	TRPMEVENT enmTrapType;
6153	switch (uVectorType)
6154	{
6155	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6156	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6157	enmTrapType = TRPM_HARDWARE_INT;
6158	break;
6159
6160	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6161	enmTrapType = TRPM_SOFTWARE_INT;
6162	break;
6163
6164	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6165	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6166	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6167	enmTrapType = TRPM_TRAP;
6168	break;
6169
6170	default:
6171	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6172	enmTrapType = TRPM_32BIT_HACK;
6173	break;
6174	}
6175
6176	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6177
6178	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6179	AssertRC(rc);
6180
6181	if (fErrorCodeValid)
6182	TRPMSetErrorCode(pVCpu, uErrorCode);
6183
6184	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6185	&& uVector == X86_XCPT_PF)
6186	{
6187	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6188	}
6189	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6190	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6191	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6192	{
6193	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6194	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6195	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6196	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6197	}
6198	pVCpu->hm.s.Event.fPending = false;
6199	}
6200
6201
6202	/**
6203	* Does the necessary state syncing before returning to ring-3 for any reason
6204	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6205	*
6206	* @returns VBox status code.
6207	* @param pVM Pointer to the VM.
6208	* @param pVCpu Pointer to the VMCPU.
6209	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6210	* be out-of-sync. Make sure to update the required
6211	* fields before using them.
6212	* @param fSaveGuestState Whether to save the guest state or not.
6213	*
6214	* @remarks If you modify code here, make sure to check whether
6215	* hmR0VmxCallRing3Callback() needs to be updated too.
6216	* @remarks No-long-jmp zone!!!
6217	*/
6218	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6219	{
6220	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6221	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6222
6223	RTCPUID idCpu = RTMpCpuId();
6224	Log4Func(("HostCpuId=%u\n", idCpu));
6225
6226	/* Save the guest state if necessary. */
6227	if ( fSaveGuestState
6228	&& pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6229	{
6230	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6231	AssertRCReturn(rc, rc);
6232	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6233	}
6234
6235	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6236	if (CPUMIsGuestFPUStateActive(pVCpu))
6237	{
6238	/* We shouldn't reload CR0 without saving it first. */
6239	if (!fSaveGuestState)
6240	{
6241	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6242	AssertRCReturn(rc, rc);
6243	}
6244	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6245	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6246	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6247	}
6248
6249	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6250	#ifdef VBOX_STRICT
6251	if (CPUMIsHyperDebugStateActive(pVCpu))
6252	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6253	#endif
6254	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6255	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6256	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6257	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6258
6259	#if HC_ARCH_BITS == 64
6260	/* Restore host-state bits that VT-x only restores partially. */
6261	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6262	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6263	{
6264	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6265	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6266	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6267	}
6268	#endif
6269
6270	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6271	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6272	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6273	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6274	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6275	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6276	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6277	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6278
6279	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6280
6281	/** @todo This kinda defeats the purpose of having preemption hooks.
6282	* The problem is, deregistering the hooks should be moved to a place that
6283	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6284	* context.
6285	*/
6286	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6287	{
6288	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6289	AssertRCReturn(rc, rc);
6290
6291	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6292	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6293	}
6294	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6295	NOREF(idCpu);
6296
6297	return VINF_SUCCESS;
6298	}
6299
6300
6301	/**
6302	* Leaves the VT-x session.
6303	*
6304	* @returns VBox status code.
6305	* @param pVM Pointer to the VM.
6306	* @param pVCpu Pointer to the VMCPU.
6307	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6308	* out-of-sync. Make sure to update the required fields
6309	* before using them.
6310	*
6311	* @remarks No-long-jmp zone!!!
6312	*/
6313	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6314	{
6315	HM_DISABLE_PREEMPT_IF_NEEDED();
6316	HMVMX_ASSERT_CPU_SAFE();
6317	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6318	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6319
6320	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6321	and done this from the VMXR0ThreadCtxCallback(). */
6322	if (!pVCpu->hm.s.fLeaveDone)
6323	{
6324	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6325	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6326	pVCpu->hm.s.fLeaveDone = true;
6327	}
6328
6329	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6330	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6331	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6332	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6333	VMMR0ThreadCtxHooksDeregister(pVCpu);
6334
6335	/* Leave HM context. This takes care of local init (term). */
6336	int rc = HMR0LeaveCpu(pVCpu);
6337
6338	HM_RESTORE_PREEMPT_IF_NEEDED();
6339
6340	return rc;
6341	}
6342
6343
6344	/**
6345	* Does the necessary state syncing before doing a longjmp to ring-3.
6346	*
6347	* @returns VBox status code.
6348	* @param pVM Pointer to the VM.
6349	* @param pVCpu Pointer to the VMCPU.
6350	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6351	* out-of-sync. Make sure to update the required fields
6352	* before using them.
6353	*
6354	* @remarks No-long-jmp zone!!!
6355	*/
6356	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6357	{
6358	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6359	}
6360
6361
6362	/**
6363	* Take necessary actions before going back to ring-3.
6364	*
6365	* An action requires us to go back to ring-3. This function does the necessary
6366	* steps before we can safely return to ring-3. This is not the same as longjmps
6367	* to ring-3, this is voluntary and prepares the guest so it may continue
6368	* executing outside HM (recompiler/IEM).
6369	*
6370	* @returns VBox status code.
6371	* @param pVM Pointer to the VM.
6372	* @param pVCpu Pointer to the VMCPU.
6373	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6374	* out-of-sync. Make sure to update the required fields
6375	* before using them.
6376	* @param rcExit The reason for exiting to ring-3. Can be
6377	* VINF_VMM_UNKNOWN_RING3_CALL.
6378	*/
6379	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6380	{
6381	Assert(pVM);
6382	Assert(pVCpu);
6383	Assert(pMixedCtx);
6384	HMVMX_ASSERT_PREEMPT_SAFE();
6385
6386	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6387	{
6388	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6389	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6390	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6391	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6392	}
6393
6394	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6395	VMMRZCallRing3Disable(pVCpu);
6396	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6397
6398	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6399	if (pVCpu->hm.s.Event.fPending)
6400	{
6401	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6402	Assert(!pVCpu->hm.s.Event.fPending);
6403	}
6404
6405	/* Save guest state and restore host state bits. */
6406	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6407	AssertRCReturn(rc, rc);
6408	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6409
6410	/* Sync recompiler state. */
6411	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6412	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6413	\| CPUM_CHANGED_LDTR
6414	\| CPUM_CHANGED_GDTR
6415	\| CPUM_CHANGED_IDTR
6416	\| CPUM_CHANGED_TR
6417	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6418	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6419	if ( pVM->hm.s.fNestedPaging
6420	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6421	{
6422	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6423	}
6424
6425	Assert(!pVCpu->hm.s.fClearTrapFlag);
6426
6427	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6428	if (rcExit != VINF_EM_RAW_INTERRUPT)
6429	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6430
6431	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6432
6433	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6434	VMMRZCallRing3RemoveNotification(pVCpu);
6435	VMMRZCallRing3Enable(pVCpu);
6436
6437	return rc;
6438	}
6439
6440
6441	/**
6442	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6443	* longjump to ring-3 and possibly get preempted.
6444	*
6445	* @returns VBox status code.
6446	* @param pVCpu Pointer to the VMCPU.
6447	* @param enmOperation The operation causing the ring-3 longjump.
6448	* @param pvUser Opaque pointer to the guest-CPU context. The data
6449	* may be out-of-sync. Make sure to update the required
6450	* fields before using them.
6451	* @remarks If you modify code here, make sure to check whether
6452	* hmR0VmxLeave() needs to be updated too.
6453	*/
6454	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6455	{
6456	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6457	{
6458	VMMRZCallRing3RemoveNotification(pVCpu);
6459	HM_DISABLE_PREEMPT_IF_NEEDED();
6460
6461	/* If anything here asserts or fails, good luck. */
6462	if (CPUMIsGuestFPUStateActive(pVCpu))
6463	CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6464
6465	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6466
6467	#if HC_ARCH_BITS == 64
6468	/* Restore host-state bits that VT-x only restores partially. */
6469	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6470	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6471	{
6472	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6473	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6474	}
6475	#endif
6476	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6477	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6478	{
6479	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6480	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6481	}
6482
6483	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6484	VMMR0ThreadCtxHooksDeregister(pVCpu);
6485
6486	HMR0LeaveCpu(pVCpu);
6487	HM_RESTORE_PREEMPT_IF_NEEDED();
6488	return VINF_SUCCESS;
6489	}
6490
6491	Assert(pVCpu);
6492	Assert(pvUser);
6493	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6494	HMVMX_ASSERT_PREEMPT_SAFE();
6495
6496	VMMRZCallRing3Disable(pVCpu);
6497	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6498
6499	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6500	enmOperation));
6501
6502	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6503	AssertRCReturn(rc, rc);
6504
6505	VMMRZCallRing3Enable(pVCpu);
6506	return VINF_SUCCESS;
6507	}
6508
6509
6510	/**
6511	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6512	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6513	*
6514	* @param pVCpu Pointer to the VMCPU.
6515	*/
6516	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6517	{
6518	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6519	{
6520	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6521	{
6522	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6523	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6524	AssertRC(rc);
6525	}
6526	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6527	}
6528
6529
6530	/**
6531	* Evaluates the event to be delivered to the guest and sets it as the pending
6532	* event.
6533	*
6534	* @param pVCpu Pointer to the VMCPU.
6535	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6536	* out-of-sync. Make sure to update the required fields
6537	* before using them.
6538	*/
6539	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6540	{
6541	Assert(!pVCpu->hm.s.Event.fPending);
6542
6543	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6544	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6545	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6546	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6547
6548	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6549	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6550	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6551	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6552	Assert(!TRPMHasTrap(pVCpu));
6553
6554	/** @todo SMI. SMIs take priority over NMIs. */
6555	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6556	{
6557	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6558	if ( !fBlockMovSS
6559	&& !fBlockSti)
6560	{
6561	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6562	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6563	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6564	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6565
6566	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6567	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6568	}
6569	else
6570	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6571	}
6572	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6573	&& !pVCpu->hm.s.fSingleInstruction)
6574	{
6575	/*
6576	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6577	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6578	* evaluated here and not set as pending, solely based on the force-flags.
6579	*/
6580	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6581	AssertRC(rc);
6582	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6583	if ( !fBlockInt
6584	&& !fBlockSti
6585	&& !fBlockMovSS)
6586	{
6587	uint8_t u8Interrupt;
6588	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6589	if (RT_SUCCESS(rc))
6590	{
6591	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6592	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6593	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6594
6595	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
6596	}
6597	else
6598	{
6599	/** @todo Does this actually happen? If not turn it into an assertion. */
6600	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6601	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6602	}
6603	}
6604	else
6605	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6606	}
6607	}
6608
6609
6610	/**
6611	* Injects any pending events into the guest if the guest is in a state to
6612	* receive them.
6613	*
6614	* @returns VBox status code (informational status codes included).
6615	* @param pVCpu Pointer to the VMCPU.
6616	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6617	* out-of-sync. Make sure to update the required fields
6618	* before using them.
6619	*/
6620	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6621	{
6622	HMVMX_ASSERT_PREEMPT_SAFE();
6623	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6624
6625	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6626	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6627	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6628	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6629
6630	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6631	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6632	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6633	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6634	Assert(!TRPMHasTrap(pVCpu));
6635
6636	int rc = VINF_SUCCESS;
6637	if (pVCpu->hm.s.Event.fPending)
6638	{
6639	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
6640	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6641	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6642	{
6643	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6644	AssertRCReturn(rc, rc);
6645	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6646	Assert(!fBlockInt);
6647	Assert(!fBlockSti);
6648	Assert(!fBlockMovSS);
6649	}
6650	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6651	{
6652	Assert(!fBlockSti);
6653	Assert(!fBlockMovSS);
6654	}
6655	#endif
6656	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
6657	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
6658	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6659	AssertRCReturn(rc, rc);
6660
6661	/* Update the interruptibility-state as it could have been changed by
6662	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6663	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6664	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6665
6666	#ifdef VBOX_WITH_STATISTICS
6667	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6668	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6669	else
6670	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6671	#endif
6672	}
6673
6674	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6675	int rc2 = VINF_SUCCESS;
6676	if ( fBlockSti
6677	\|\| fBlockMovSS)
6678	{
6679	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6680	{
6681	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6682	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6683	{
6684	/*
6685	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6686	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6687	* See Intel spec. 27.3.4 "Saving Non-Register State".
6688	*/
6689	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6690	AssertRCReturn(rc, rc);
6691	}
6692	}
6693	else
6694	{
6695	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6696	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6697	uIntrState = 0;
6698	}
6699	}
6700
6701	/*
6702	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6703	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6704	*/
6705	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6706	AssertRC(rc2);
6707
6708	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6709	return rc;
6710	}
6711
6712
6713	/**
6714	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6715	*
6716	* @param pVCpu Pointer to the VMCPU.
6717	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6718	* out-of-sync. Make sure to update the required fields
6719	* before using them.
6720	*/
6721	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6722	{
6723	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6724	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6725	}
6726
6727
6728	/**
6729	* Injects a double-fault (#DF) exception into the VM.
6730	*
6731	* @returns VBox status code (informational status code included).
6732	* @param pVCpu Pointer to the VMCPU.
6733	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6734	* out-of-sync. Make sure to update the required fields
6735	* before using them.
6736	*/
6737	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6738	{
6739	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6740	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6741	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6742	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6743	puIntrState);
6744	}
6745
6746
6747	/**
6748	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6749	*
6750	* @param pVCpu Pointer to the VMCPU.
6751	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6752	* out-of-sync. Make sure to update the required fields
6753	* before using them.
6754	*/
6755	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6756	{
6757	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6758	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6759	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6760	}
6761
6762
6763	/**
6764	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6765	*
6766	* @param pVCpu Pointer to the VMCPU.
6767	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6768	* out-of-sync. Make sure to update the required fields
6769	* before using them.
6770	* @param cbInstr The value of RIP that is to be pushed on the guest
6771	* stack.
6772	*/
6773	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6774	{
6775	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6776	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6777	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6778	}
6779
6780
6781	/**
6782	* Injects a general-protection (#GP) fault into the VM.
6783	*
6784	* @returns VBox status code (informational status code included).
6785	* @param pVCpu Pointer to the VMCPU.
6786	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6787	* out-of-sync. Make sure to update the required fields
6788	* before using them.
6789	* @param u32ErrorCode The error code associated with the #GP.
6790	*/
6791	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6792	uint32_t *puIntrState)
6793	{
6794	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6795	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6796	if (fErrorCodeValid)
6797	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6798	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6799	puIntrState);
6800	}
6801
6802
6803	/**
6804	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6805	*
6806	* @param pVCpu Pointer to the VMCPU.
6807	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6808	* out-of-sync. Make sure to update the required fields
6809	* before using them.
6810	* @param uVector The software interrupt vector number.
6811	* @param cbInstr The value of RIP that is to be pushed on the guest
6812	* stack.
6813	*/
6814	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6815	{
6816	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6817	if ( uVector == X86_XCPT_BP
6818	\|\| uVector == X86_XCPT_OF)
6819	{
6820	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6821	}
6822	else
6823	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6824	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6825	}
6826
6827
6828	/**
6829	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6830	* stack.
6831	*
6832	* @returns VBox status code (information status code included).
6833	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6834	* @param pVM Pointer to the VM.
6835	* @param pMixedCtx Pointer to the guest-CPU context.
6836	* @param uValue The value to push to the guest stack.
6837	*/
6838	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6839	{
6840	/*
6841	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6842	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6843	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6844	*/
6845	if (pMixedCtx->sp == 1)
6846	return VINF_EM_RESET;
6847	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6848	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6849	AssertRCReturn(rc, rc);
6850	return rc;
6851	}
6852
6853
6854	/**
6855	* Injects an event into the guest upon VM-entry by updating the relevant fields
6856	* in the VM-entry area in the VMCS.
6857	*
6858	* @returns VBox status code (informational error codes included).
6859	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6860	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6861	*
6862	* @param pVCpu Pointer to the VMCPU.
6863	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6864	* be out-of-sync. Make sure to update the required
6865	* fields before using them.
6866	* @param u64IntInfo The VM-entry interruption-information field.
6867	* @param cbInstr The VM-entry instruction length in bytes (for
6868	* software interrupts, exceptions and privileged
6869	* software exceptions).
6870	* @param u32ErrCode The VM-entry exception error code.
6871	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6872	* @param puIntrState Pointer to the current guest interruptibility-state.
6873	* This interruptibility-state will be updated if
6874	* necessary. This cannot not be NULL.
6875	*
6876	* @remarks Requires CR0!
6877	* @remarks No-long-jump zone!!!
6878	*/
6879	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
6880	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6881	{
6882	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6883	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
6884	Assert(puIntrState);
6885	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
6886
6887	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
6888	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
6889
6890	#ifdef VBOX_STRICT
6891	/* Validate the error-code-valid bit for hardware exceptions. */
6892	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6893	{
6894	switch (uVector)
6895	{
6896	case X86_XCPT_PF:
6897	case X86_XCPT_DF:
6898	case X86_XCPT_TS:
6899	case X86_XCPT_NP:
6900	case X86_XCPT_SS:
6901	case X86_XCPT_GP:
6902	case X86_XCPT_AC:
6903	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
6904	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6905	/* fallthru */
6906	default:
6907	break;
6908	}
6909	}
6910	#endif
6911
6912	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6913	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6914	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6915
6916	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6917
6918	/* We require CR0 to check if the guest is in real-mode. */
6919	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6920	AssertRCReturn(rc, rc);
6921
6922	/*
6923	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6924	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6925	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6926	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6927	*/
6928	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6929	{
6930	PVM pVM = pVCpu->CTX_SUFF(pVM);
6931	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6932	{
6933	Assert(PDMVmmDevHeapIsEnabled(pVM));
6934	Assert(pVM->hm.s.vmx.pRealModeTSS);
6935
6936	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6937	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6938	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6939	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6940	AssertRCReturn(rc, rc);
6941	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6942
6943	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6944	const size_t cbIdtEntry = sizeof(X86IDTR16);
6945	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6946	{
6947	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6948	if (uVector == X86_XCPT_DF)
6949	return VINF_EM_RESET;
6950	else if (uVector == X86_XCPT_GP)
6951	{
6952	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6953	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6954	}
6955
6956	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6957	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6958	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6959	}
6960
6961	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6962	uint16_t uGuestIp = pMixedCtx->ip;
6963	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6964	{
6965	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6966	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6967	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6968	}
6969	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6970	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6971
6972	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6973	X86IDTR16 IdtEntry;
6974	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6975	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6976	AssertRCReturn(rc, rc);
6977
6978	/* Construct the stack frame for the interrupt/exception handler. */
6979	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6980	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6981	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6982	AssertRCReturn(rc, rc);
6983
6984	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6985	if (rc == VINF_SUCCESS)
6986	{
6987	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6988	pMixedCtx->rip = IdtEntry.offSel;
6989	pMixedCtx->cs.Sel = IdtEntry.uSel;
6990	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
6991	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6992	&& uVector == X86_XCPT_PF)
6993	{
6994	pMixedCtx->cr2 = GCPtrFaultAddress;
6995	}
6996
6997	/* If any other guest-state bits are changed here, make sure to update
6998	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6999	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7000	\| HM_CHANGED_GUEST_RIP
7001	\| HM_CHANGED_GUEST_RFLAGS
7002	\| HM_CHANGED_GUEST_RSP);
7003
7004	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7005	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7006	{
7007	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7008	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7009	Log4(("Clearing inhibition due to STI.\n"));
7010	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7011	}
7012	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7013
7014	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7015	it, if we are returning to ring-3 before executing guest code. */
7016	pVCpu->hm.s.Event.fPending = false;
7017	}
7018	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7019	return rc;
7020	}
7021	else
7022	{
7023	/*
7024	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7025	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7026	*/
7027	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7028	}
7029	}
7030
7031	/* Validate. */
7032	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7033	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7034	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7035
7036	/* Inject. */
7037	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7038	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7039	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7040	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7041
7042	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7043	&& uVector == X86_XCPT_PF)
7044	{
7045	pMixedCtx->cr2 = GCPtrFaultAddress;
7046	}
7047
7048	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7049	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7050
7051	AssertRCReturn(rc, rc);
7052	return rc;
7053	}
7054
7055
7056	/**
7057	* Clears the interrupt-window exiting control in the VMCS and if necessary
7058	* clears the current event in the VMCS as well.
7059	*
7060	* @returns VBox status code.
7061	* @param pVCpu Pointer to the VMCPU.
7062	*
7063	* @remarks Use this function only to clear events that have not yet been
7064	* delivered to the guest but are injected in the VMCS!
7065	* @remarks No-long-jump zone!!!
7066	*/
7067	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7068	{
7069	int rc;
7070	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7071
7072	/* Clear interrupt-window exiting control. */
7073	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7074	{
7075	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7076	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7077	AssertRC(rc);
7078	}
7079
7080	if (!pVCpu->hm.s.Event.fPending)
7081	return;
7082
7083	#ifdef VBOX_STRICT
7084	uint32_t u32EntryInfo;
7085	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7086	AssertRC(rc);
7087	Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
7088	#endif
7089
7090	/* Clear the entry-interruption field (including the valid bit). */
7091	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7092	AssertRC(rc);
7093
7094	/* Clear the pending debug exception field. */
7095	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7096	AssertRC(rc);
7097	}
7098
7099
7100	/**
7101	* Enters the VT-x session.
7102	*
7103	* @returns VBox status code.
7104	* @param pVM Pointer to the VM.
7105	* @param pVCpu Pointer to the VMCPU.
7106	* @param pCpu Pointer to the CPU info struct.
7107	*/
7108	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7109	{
7110	AssertPtr(pVM);
7111	AssertPtr(pVCpu);
7112	Assert(pVM->hm.s.vmx.fSupported);
7113	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7114	NOREF(pCpu);
7115
7116	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7117	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7118
7119	#ifdef VBOX_STRICT
7120	/* Make sure we're in VMX root mode. */
7121	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7122	if (!(u32HostCR4 & X86_CR4_VMXE))
7123	{
7124	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7125	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7126	}
7127	#endif
7128
7129	/*
7130	* Load the VCPU's VMCS as the current (and active) one.
7131	*/
7132	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7133	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7134	if (RT_FAILURE(rc))
7135	return rc;
7136
7137	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7138	pVCpu->hm.s.fLeaveDone = false;
7139	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7140
7141	return VINF_SUCCESS;
7142	}
7143
7144
7145	/**
7146	* The thread-context callback (only on platforms which support it).
7147	*
7148	* @param enmEvent The thread-context event.
7149	* @param pVCpu Pointer to the VMCPU.
7150	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7151	* @thread EMT(pVCpu)
7152	*/
7153	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7154	{
7155	switch (enmEvent)
7156	{
7157	case RTTHREADCTXEVENT_PREEMPTING:
7158	{
7159	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7160	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7161	VMCPU_ASSERT_EMT(pVCpu);
7162
7163	PVM pVM = pVCpu->CTX_SUFF(pVM);
7164	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7165
7166	/* No longjmps (logger flushes, locks) in this fragile context. */
7167	VMMRZCallRing3Disable(pVCpu);
7168	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7169
7170	/*
7171	* Restore host-state (FPU, debug etc.)
7172	*/
7173	if (!pVCpu->hm.s.fLeaveDone)
7174	{
7175	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7176	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7177	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7178	pVCpu->hm.s.fLeaveDone = true;
7179	}
7180
7181	/* Leave HM context, takes care of local init (term). */
7182	int rc = HMR0LeaveCpu(pVCpu);
7183	AssertRC(rc); NOREF(rc);
7184
7185	/* Restore longjmp state. */
7186	VMMRZCallRing3Enable(pVCpu);
7187	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7188	break;
7189	}
7190
7191	case RTTHREADCTXEVENT_RESUMED:
7192	{
7193	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7194	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7195	VMCPU_ASSERT_EMT(pVCpu);
7196
7197	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7198	VMMRZCallRing3Disable(pVCpu);
7199	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7200
7201	/* Initialize the bare minimum state required for HM. This takes care of
7202	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7203	int rc = HMR0EnterCpu(pVCpu);
7204	AssertRC(rc);
7205	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7206
7207	/* Load the active VMCS as the current one. */
7208	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7209	{
7210	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7211	AssertRC(rc); NOREF(rc);
7212	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7213	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7214	}
7215	pVCpu->hm.s.fLeaveDone = false;
7216
7217	/* Restore longjmp state. */
7218	VMMRZCallRing3Enable(pVCpu);
7219	break;
7220	}
7221
7222	default:
7223	break;
7224	}
7225	}
7226
7227
7228	/**
7229	* Saves the host state in the VMCS host-state.
7230	* Sets up the VM-exit MSR-load area.
7231	*
7232	* The CPU state will be loaded from these fields on every successful VM-exit.
7233	*
7234	* @returns VBox status code.
7235	* @param pVM Pointer to the VM.
7236	* @param pVCpu Pointer to the VMCPU.
7237	*
7238	* @remarks No-long-jump zone!!!
7239	*/
7240	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7241	{
7242	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7243
7244	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7245	return VINF_SUCCESS;
7246
7247	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7248	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7249
7250	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7251	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7252
7253	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7254	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7255
7256	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7257	return rc;
7258	}
7259
7260
7261	/**
7262	* Saves the host state in the VMCS host-state.
7263	*
7264	* @returns VBox status code.
7265	* @param pVM Pointer to the VM.
7266	* @param pVCpu Pointer to the VMCPU.
7267	*
7268	* @remarks No-long-jump zone!!!
7269	*/
7270	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7271	{
7272	AssertPtr(pVM);
7273	AssertPtr(pVCpu);
7274
7275	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7276
7277	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7278	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7279	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7280	return hmR0VmxSaveHostState(pVM, pVCpu);
7281	}
7282
7283
7284	/**
7285	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7286	* loaded from these fields on every successful VM-entry.
7287	*
7288	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7289	* Sets up the VM-entry controls.
7290	* Sets up the appropriate VMX non-root function to execute guest code based on
7291	* the guest CPU mode.
7292	*
7293	* @returns VBox status code.
7294	* @param pVM Pointer to the VM.
7295	* @param pVCpu Pointer to the VMCPU.
7296	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7297	* out-of-sync. Make sure to update the required fields
7298	* before using them.
7299	*
7300	* @remarks No-long-jump zone!!!
7301	*/
7302	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7303	{
7304	AssertPtr(pVM);
7305	AssertPtr(pVCpu);
7306	AssertPtr(pMixedCtx);
7307	HMVMX_ASSERT_PREEMPT_SAFE();
7308
7309	#ifdef LOG_ENABLED
7310	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7311	* probably not initialized yet? Anyway this will do for now.
7312	*
7313	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7314	* interface and disable ring-3 calls when thread-context hooks are not
7315	* available. */
7316	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7317	VMMR0LogFlushDisable(pVCpu);
7318	#endif
7319
7320	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7321
7322	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7323
7324	/* Determine real-on-v86 mode. */
7325	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7326	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7327	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7328	{
7329	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7330	}
7331
7332	/*
7333	* Load the guest-state into the VMCS.
7334	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7335	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7336	*/
7337	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7338	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7339
7340	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7341	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7342	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7343
7344	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7345	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7346	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7347
7348	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7349	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7350
7351	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7352	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7353
7354	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7355	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7356	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7357
7358	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7359	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7360
7361	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7362	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7363
7364	/*
7365	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7366	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7367	*/
7368	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7369	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7370
7371	/* Clear any unused and reserved bits. */
7372	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7373
7374	#ifdef LOG_ENABLED
7375	/* Only reenable log-flushing if the caller has it enabled. */
7376	if (!fCallerDisabledLogFlush)
7377	VMMR0LogFlushEnable(pVCpu);
7378	#endif
7379
7380	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7381	return rc;
7382	}
7383
7384
7385	/**
7386	* Loads the state shared between the host and guest into the VMCS.
7387	*
7388	* @param pVM Pointer to the VM.
7389	* @param pVCpu Pointer to the VMCPU.
7390	* @param pCtx Pointer to the guest-CPU context.
7391	*
7392	* @remarks No-long-jump zone!!!
7393	*/
7394	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7395	{
7396	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7397	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7398
7399	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7400	{
7401	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7402	AssertRC(rc);
7403	}
7404
7405	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7406	{
7407	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7408	AssertRC(rc);
7409
7410	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7411	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7412	{
7413	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7414	AssertRC(rc);
7415	}
7416	}
7417
7418	AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7419	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7420	}
7421
7422
7423	/**
7424	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7425	*
7426	* @param pVM Pointer to the VM.
7427	* @param pVCpu Pointer to the VMCPU.
7428	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7429	* out-of-sync. Make sure to update the required fields
7430	* before using them.
7431	*/
7432	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7433	{
7434	HMVMX_ASSERT_PREEMPT_SAFE();
7435
7436	Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7437	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7438	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7439	#endif
7440
7441	if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7442	{
7443	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7444	AssertRC(rc);
7445	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7446	}
7447	else if (VMCPU_HMCF_VALUE(pVCpu))
7448	{
7449	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7450	AssertRC(rc);
7451	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7452	}
7453
7454	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7455	AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7456	\|\| VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7457	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7458
7459	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7460	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7461	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7462	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7463	#endif
7464	}
7465
7466
7467	/**
7468	* Does the preparations before executing guest code in VT-x.
7469	*
7470	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7471	* recompiler. We must be cautious what we do here regarding committing
7472	* guest-state information into the VMCS assuming we assuredly execute the
7473	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7474	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7475	* so that the recompiler can (and should) use them when it resumes guest
7476	* execution. Otherwise such operations must be done when we can no longer
7477	* exit to ring-3.
7478	*
7479	* @returns Strict VBox status code.
7480	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7481	* have been disabled.
7482	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7483	* double-fault into the guest.
7484	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7485	*
7486	* @param pVM Pointer to the VM.
7487	* @param pVCpu Pointer to the VMCPU.
7488	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7489	* out-of-sync. Make sure to update the required fields
7490	* before using them.
7491	* @param pVmxTransient Pointer to the VMX transient structure.
7492	*
7493	* @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7494	* interrupts will be disabled.
7495	*/
7496	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7497	{
7498	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7499
7500	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7501	PGMRZDynMapFlushAutoSet(pVCpu);
7502	#endif
7503
7504	/* Check force flag actions that might require us to go back to ring-3. */
7505	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7506	if (rc != VINF_SUCCESS)
7507	return rc;
7508
7509	#ifndef IEM_VERIFICATION_MODE_FULL
7510	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7511	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7512	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7513	{
7514	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7515	RTGCPHYS GCPhysApicBase;
7516	GCPhysApicBase = pMixedCtx->msrApicBase;
7517	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7518
7519	/* Unalias any existing mapping. */
7520	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7521	AssertRCReturn(rc, rc);
7522
7523	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7524	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7525	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7526	AssertRCReturn(rc, rc);
7527
7528	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7529	}
7530	#endif /* !IEM_VERIFICATION_MODE_FULL */
7531
7532	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7533	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7534
7535	/*
7536	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7537	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7538	*/
7539	if (TRPMHasTrap(pVCpu))
7540	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7541	else if (!pVCpu->hm.s.Event.fPending)
7542	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7543
7544	/*
7545	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7546	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7547	*/
7548	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7549	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7550	{
7551	Assert(rc == VINF_EM_RESET);
7552	return rc;
7553	}
7554
7555	/*
7556	* No longjmps to ring-3 from this point on!!!
7557	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7558	* This also disables flushing of the R0-logger instance (if any).
7559	*/
7560	VMMRZCallRing3Disable(pVCpu);
7561
7562	/*
7563	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7564	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7565	*
7566	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7567	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7568	*
7569	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7570	* executing guest code.
7571	*/
7572	pVmxTransient->uEflags = ASMIntDisableFlags();
7573	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7574	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7575	{
7576	hmR0VmxClearEventVmcs(pVCpu);
7577	ASMSetFlags(pVmxTransient->uEflags);
7578	VMMRZCallRing3Enable(pVCpu);
7579	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7580	return VINF_EM_RAW_TO_R3;
7581	}
7582	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7583	{
7584	hmR0VmxClearEventVmcs(pVCpu);
7585	ASMSetFlags(pVmxTransient->uEflags);
7586	VMMRZCallRing3Enable(pVCpu);
7587	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7588	return VINF_EM_RAW_INTERRUPT;
7589	}
7590
7591	/* We've injected any pending events. This is really the point of no return (to ring-3). */
7592	pVCpu->hm.s.Event.fPending = false;
7593
7594	return VINF_SUCCESS;
7595	}
7596
7597
7598	/**
7599	* Prepares to run guest code in VT-x and we've committed to doing so. This
7600	* means there is no backing out to ring-3 or anywhere else at this
7601	* point.
7602	*
7603	* @param pVM Pointer to the VM.
7604	* @param pVCpu Pointer to the VMCPU.
7605	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7606	* out-of-sync. Make sure to update the required fields
7607	* before using them.
7608	* @param pVmxTransient Pointer to the VMX transient structure.
7609	*
7610	* @remarks Called with preemption disabled.
7611	* @remarks No-long-jump zone!!!
7612	*/
7613	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7614	{
7615	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7616	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7617	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7618
7619	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7620	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7621
7622	/*
7623	* If we are injecting events to a real-on-v86 mode guest, we may have to update
7624	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7625	* Reload only the necessary state, the assertion will catch if other parts of the code
7626	* change.
7627	*/
7628	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7629	{
7630	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7631	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7632	}
7633
7634	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7635	if (!CPUMIsGuestFPUStateActive(pVCpu))
7636	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7637	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7638	#endif
7639
7640	/*
7641	* Load the host state bits as we may've been preempted (only happens when
7642	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7643	*/
7644	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7645	{
7646	/* This ASSUMES that pfnStartVM has been set up already. */
7647	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7648	AssertRC(rc);
7649	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7650	}
7651	Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
7652
7653	/*
7654	* Load the state shared between host and guest (FPU, debug).
7655	*/
7656	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
7657	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7658	AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7659
7660	/* Store status of the shared guest-host state at the time of VM-entry. */
7661	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7662	if (CPUMIsGuestInLongModeEx(pMixedCtx))
7663	{
7664	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7665	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7666	}
7667	else
7668	#endif
7669	{
7670	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7671	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7672	}
7673	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7674
7675	/*
7676	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7677	*/
7678	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7679	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7680
7681	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7682	RTCPUID idCurrentCpu = pCpu->idCpu;
7683	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7684	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
7685	{
7686	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7687	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7688	}
7689
7690	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7691	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7692	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7693	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7694
7695	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7696
7697	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7698	to start executing. */
7699
7700	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7701	/*
7702	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7703	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7704	*/
7705	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7706	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7707	{
7708	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7709	uint64_t u64HostTscAux = 0;
7710	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7711	AssertRC(rc2);
7712	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7713	}
7714	#endif
7715	}
7716
7717
7718	/**
7719	* Performs some essential restoration of state after running guest code in
7720	* VT-x.
7721	*
7722	* @param pVM Pointer to the VM.
7723	* @param pVCpu Pointer to the VMCPU.
7724	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7725	* out-of-sync. Make sure to update the required fields
7726	* before using them.
7727	* @param pVmxTransient Pointer to the VMX transient structure.
7728	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7729	*
7730	* @remarks Called with interrupts disabled, and returns with interrups enabled!
7731	*
7732	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7733	* unconditionally when it is safe to do so.
7734	*/
7735	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7736	{
7737	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7738
7739	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7740	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7741	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7742	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7743	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7744
7745	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7746	{
7747	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7748	/* Restore host's TSC_AUX. */
7749	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7750	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7751	#endif
7752	/** @todo Find a way to fix hardcoding a guestimate. */
7753	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7754	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7755	}
7756
7757	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7758	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7759	Assert(!(ASMGetFlags() & X86_EFL_IF));
7760	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7761
7762	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7763	if (CPUMIsGuestFPUStateActive(pVCpu))
7764	{
7765	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7766	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7767	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7768	}
7769	#endif
7770
7771	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7772	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7773	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7774	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7775
7776	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7777	uint32_t uExitReason;
7778	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7779	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
7780	AssertRC(rc);
7781	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7782	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntInfo);
7783
7784	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7785	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7786	{
7787	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7788	pVmxTransient->fVMEntryFailed));
7789	return;
7790	}
7791
7792	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7793	{
7794	/* Update the guest interruptibility-state from the VMCS. */
7795	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7796	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7797	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7798	AssertRC(rc);
7799	#endif
7800	/*
7801	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7802	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7803	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7804	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7805	*/
7806	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7807	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7808	{
7809	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7810	AssertRC(rc);
7811	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
7812	}
7813	}
7814	}
7815
7816
7817
7818	/**
7819	* Runs the guest code using VT-x the normal way.
7820	*
7821	* @returns VBox status code.
7822	* @param pVM Pointer to the VM.
7823	* @param pVCpu Pointer to the VMCPU.
7824	* @param pCtx Pointer to the guest-CPU context.
7825	*
7826	* @note Mostly the same as hmR0VmxRunGuestCodeStep.
7827	* @remarks Called with preemption disabled.
7828	*/
7829	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7830	{
7831	VMXTRANSIENT VmxTransient;
7832	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7833	int rc = VERR_INTERNAL_ERROR_5;
7834	uint32_t cLoops = 0;
7835
7836	for (;; cLoops++)
7837	{
7838	Assert(!HMR0SuspendPending());
7839	HMVMX_ASSERT_CPU_SAFE();
7840
7841	/* Preparatory work for running guest code, this may force us to return
7842	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7843	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7844	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7845	if (rc != VINF_SUCCESS)
7846	break;
7847
7848	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7849	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7850	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7851
7852	/* Restore any residual host-state and save any bits shared between host
7853	and guest into the guest-CPU state. Re-enables interrupts! */
7854	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7855
7856	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7857	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7858	{
7859	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7860	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7861	return rc;
7862	}
7863
7864	/* Handle the VM-exit. */
7865	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7866	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7867	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7868	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7869	HMVMX_START_EXIT_DISPATCH_PROF();
7870	#ifdef HMVMX_USE_FUNCTION_TABLE
7871	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7872	#else
7873	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7874	#endif
7875	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7876	if (rc != VINF_SUCCESS)
7877	break;
7878	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7879	{
7880	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7881	rc = VINF_EM_RAW_INTERRUPT;
7882	break;
7883	}
7884	}
7885
7886	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7887	return rc;
7888	}
7889
7890
7891	/**
7892	* Single steps guest code using VT-x.
7893	*
7894	* @returns VBox status code.
7895	* @param pVM Pointer to the VM.
7896	* @param pVCpu Pointer to the VMCPU.
7897	* @param pCtx Pointer to the guest-CPU context.
7898	*
7899	* @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7900	* @remarks Called with preemption disabled.
7901	*/
7902	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7903	{
7904	VMXTRANSIENT VmxTransient;
7905	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7906	int rc = VERR_INTERNAL_ERROR_5;
7907	uint32_t cLoops = 0;
7908	uint16_t uCsStart = pCtx->cs.Sel;
7909	uint64_t uRipStart = pCtx->rip;
7910
7911	for (;; cLoops++)
7912	{
7913	Assert(!HMR0SuspendPending());
7914	HMVMX_ASSERT_CPU_SAFE();
7915
7916	/* Preparatory work for running guest code, this may force us to return
7917	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7918	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7919	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7920	if (rc != VINF_SUCCESS)
7921	break;
7922
7923	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7924	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7925	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7926
7927	/* Restore any residual host-state and save any bits shared between host
7928	and guest into the guest-CPU state. Re-enables interrupts! */
7929	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7930
7931	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7932	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7933	{
7934	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7935	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7936	return rc;
7937	}
7938
7939	/* Handle the VM-exit. */
7940	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7941	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7942	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7943	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7944	HMVMX_START_EXIT_DISPATCH_PROF();
7945	#ifdef HMVMX_USE_FUNCTION_TABLE
7946	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7947	#else
7948	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7949	#endif
7950	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7951	if (rc != VINF_SUCCESS)
7952	break;
7953	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7954	{
7955	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7956	rc = VINF_EM_RAW_INTERRUPT;
7957	break;
7958	}
7959
7960	/*
7961	* Did the RIP change, if so, consider it a single step.
7962	* Otherwise, make sure one of the TFs gets set.
7963	*/
7964	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7965	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7966	AssertRCReturn(rc2, rc2);
7967	if ( pCtx->rip != uRipStart
7968	\|\| pCtx->cs.Sel != uCsStart)
7969	{
7970	rc = VINF_EM_DBG_STEPPED;
7971	break;
7972	}
7973	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7974	}
7975
7976	/*
7977	* Clear the X86_EFL_TF if necessary.
7978	*/
7979	if (pVCpu->hm.s.fClearTrapFlag)
7980	{
7981	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
7982	AssertRCReturn(rc2, rc2);
7983	pVCpu->hm.s.fClearTrapFlag = false;
7984	pCtx->eflags.Bits.u1TF = 0;
7985	}
7986	/** @todo there seems to be issues with the resume flag when the monitor trap
7987	* flag is pending without being used. Seen early in bios init when
7988	* accessing APIC page in prot mode. */
7989
7990	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7991	return rc;
7992	}
7993
7994
7995	/**
7996	* Runs the guest code using VT-x.
7997	*
7998	* @returns VBox status code.
7999	* @param pVM Pointer to the VM.
8000	* @param pVCpu Pointer to the VMCPU.
8001	* @param pCtx Pointer to the guest-CPU context.
8002	*
8003	* @remarks Called with preemption disabled.
8004	*/
8005	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8006	{
8007	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8008	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8009	HMVMX_ASSERT_PREEMPT_SAFE();
8010
8011	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8012
8013	int rc;
8014	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8015	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8016	else
8017	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8018
8019	if (rc == VERR_EM_INTERPRETER)
8020	rc = VINF_EM_RAW_EMULATE_INSTR;
8021	else if (rc == VINF_EM_RESET)
8022	rc = VINF_EM_TRIPLE_FAULT;
8023
8024	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8025	if (RT_FAILURE(rc2))
8026	{
8027	pVCpu->hm.s.u32HMError = rc;
8028	rc = rc2;
8029	}
8030	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8031	return rc;
8032	}
8033
8034
8035	#ifndef HMVMX_USE_FUNCTION_TABLE
8036	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8037	{
8038	int rc;
8039	switch (rcReason)
8040	{
8041	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
8042	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
8043	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
8044	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
8045	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
8046	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
8047	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8048	case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
8049	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
8050	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
8051	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8052	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
8053	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
8054	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
8055	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
8056	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8057	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8058	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
8059	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
8060	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
8061	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
8062	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
8063	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
8064	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
8065	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
8066	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8067	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8068	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
8069	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
8070	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
8071	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
8072	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
8073	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
8074
8075	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8076	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8077	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8078	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8079	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8080	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8081	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8082	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8083	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8084
8085	case VMX_EXIT_VMCALL:
8086	case VMX_EXIT_VMCLEAR:
8087	case VMX_EXIT_VMLAUNCH:
8088	case VMX_EXIT_VMPTRLD:
8089	case VMX_EXIT_VMPTRST:
8090	case VMX_EXIT_VMREAD:
8091	case VMX_EXIT_VMRESUME:
8092	case VMX_EXIT_VMWRITE:
8093	case VMX_EXIT_VMXOFF:
8094	case VMX_EXIT_VMXON:
8095	case VMX_EXIT_INVEPT:
8096	case VMX_EXIT_INVVPID:
8097	case VMX_EXIT_VMFUNC:
8098	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8099	break;
8100	default:
8101	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8102	break;
8103	}
8104	return rc;
8105	}
8106	#endif
8107
8108	#ifdef DEBUG
8109	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8110	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8111	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8112
8113	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8114	do \
8115	{ \
8116	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8117	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8118	} while (0)
8119
8120	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8121	do { \
8122	AssertPtr(pVCpu); \
8123	AssertPtr(pMixedCtx); \
8124	AssertPtr(pVmxTransient); \
8125	Assert(pVmxTransient->fVMEntryFailed == false); \
8126	Assert(ASMIntAreEnabled()); \
8127	HMVMX_ASSERT_PREEMPT_SAFE(); \
8128	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8129	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8130	HMVMX_ASSERT_PREEMPT_SAFE(); \
8131	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8132	HMVMX_ASSERT_PREEMPT_CPUID(); \
8133	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8134	} while (0)
8135
8136	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8137	do { \
8138	Log4Func(("\n")); \
8139	} while(0)
8140	#else /* Release builds */
8141	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8142	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8143	#endif
8144
8145
8146	/**
8147	* Advances the guest RIP after reading it from the VMCS.
8148	*
8149	* @returns VBox status code.
8150	* @param pVCpu Pointer to the VMCPU.
8151	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8152	* out-of-sync. Make sure to update the required fields
8153	* before using them.
8154	* @param pVmxTransient Pointer to the VMX transient structure.
8155	*
8156	* @remarks No-long-jump zone!!!
8157	*/
8158	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8159	{
8160	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8161	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8162	AssertRCReturn(rc, rc);
8163
8164	pMixedCtx->rip += pVmxTransient->cbInstr;
8165	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8166	return rc;
8167	}
8168
8169
8170	/**
8171	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8172	* and update error record fields accordingly.
8173	*
8174	* @return VMX_IGS_* return codes.
8175	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8176	* wrong with the guest state.
8177	*
8178	* @param pVM Pointer to the VM.
8179	* @param pVCpu Pointer to the VMCPU.
8180	* @param pCtx Pointer to the guest-CPU state.
8181	*/
8182	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8183	{
8184	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8185	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8186	uError = (err); \
8187	break; \
8188	} else do {} while (0)
8189	/* Duplicate of IEM_IS_CANONICAL(). */
8190	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8191
8192	int rc;
8193	uint32_t uError = VMX_IGS_ERROR;
8194	uint32_t u32Val;
8195	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8196
8197	do
8198	{
8199	/*
8200	* CR0.
8201	*/
8202	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8203	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8204	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8205	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8206	if (fUnrestrictedGuest)
8207	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8208
8209	uint32_t u32GuestCR0;
8210	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8211	AssertRCBreak(rc);
8212	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8213	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8214	if ( !fUnrestrictedGuest
8215	&& (u32GuestCR0 & X86_CR0_PG)
8216	&& !(u32GuestCR0 & X86_CR0_PE))
8217	{
8218	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8219	}
8220
8221	/*
8222	* CR4.
8223	*/
8224	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8225	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8226
8227	uint32_t u32GuestCR4;
8228	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8229	AssertRCBreak(rc);
8230	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8231	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8232
8233	/*
8234	* IA32_DEBUGCTL MSR.
8235	*/
8236	uint64_t u64Val;
8237	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8238	AssertRCBreak(rc);
8239	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8240	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8241	{
8242	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8243	}
8244	uint64_t u64DebugCtlMsr = u64Val;
8245
8246	#ifdef VBOX_STRICT
8247	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8248	AssertRCBreak(rc);
8249	Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
8250	#endif
8251	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8252
8253	/*
8254	* RIP and RFLAGS.
8255	*/
8256	uint32_t u32Eflags;
8257	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8258	if (HMVMX_IS_64BIT_HOST_MODE())
8259	{
8260	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8261	AssertRCBreak(rc);
8262	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8263	if ( !fLongModeGuest
8264	\|\| !pCtx->cs.Attr.n.u1Long)
8265	{
8266	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8267	}
8268	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8269	* must be identical if the "IA32e mode guest" VM-entry control is 1
8270	* and CS.L is 1. No check applies if the CPU supports 64
8271	* linear-address bits. */
8272
8273	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8274	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8275	AssertRCBreak(rc);
8276	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8277	VMX_IGS_RFLAGS_RESERVED);
8278	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8279	u32Eflags = u64Val;
8280	}
8281	else
8282	#endif
8283	{
8284	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8285	AssertRCBreak(rc);
8286	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8287	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8288	}
8289
8290	if ( fLongModeGuest
8291	\|\| ( fUnrestrictedGuest
8292	&& !(u32GuestCR0 & X86_CR0_PE)))
8293	{
8294	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8295	}
8296
8297	uint32_t u32EntryInfo;
8298	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8299	AssertRCBreak(rc);
8300	if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8301	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8302	{
8303	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8304	}
8305
8306	/*
8307	* 64-bit checks.
8308	*/
8309	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8310	if (HMVMX_IS_64BIT_HOST_MODE())
8311	{
8312	if ( fLongModeGuest
8313	&& !fUnrestrictedGuest)
8314	{
8315	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8316	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8317	}
8318
8319	if ( !fLongModeGuest
8320	&& (u32GuestCR4 & X86_CR4_PCIDE))
8321	{
8322	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8323	}
8324
8325	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8326	* 51:32 beyond the processor's physical-address width are 0. */
8327
8328	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8329	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8330	{
8331	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8332	}
8333
8334	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8335	AssertRCBreak(rc);
8336	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8337
8338	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8339	AssertRCBreak(rc);
8340	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8341	}
8342	#endif
8343
8344	/*
8345	* PERF_GLOBAL MSR.
8346	*/
8347	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8348	{
8349	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8350	AssertRCBreak(rc);
8351	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8352	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8353	}
8354
8355	/*
8356	* PAT MSR.
8357	*/
8358	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8359	{
8360	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8361	AssertRCBreak(rc);
8362	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8363	for (unsigned i = 0; i < 8; i++)
8364	{
8365	uint8_t u8Val = (u64Val & 0x7);
8366	if ( u8Val != 0 /* UC */
8367	\|\| u8Val != 1 /* WC */
8368	\|\| u8Val != 4 /* WT */
8369	\|\| u8Val != 5 /* WP */
8370	\|\| u8Val != 6 /* WB */
8371	\|\| u8Val != 7 /* UC- */)
8372	{
8373	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8374	}
8375	u64Val >>= 3;
8376	}
8377	}
8378
8379	/*
8380	* EFER MSR.
8381	*/
8382	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8383	{
8384	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8385	AssertRCBreak(rc);
8386	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8387	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8388	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8389	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8390	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8391	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8392	}
8393
8394	/*
8395	* Segment registers.
8396	*/
8397	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8398	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8399	if (!(u32Eflags & X86_EFL_VM))
8400	{
8401	/* CS */
8402	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8403	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8404	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8405	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8406	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8407	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8408	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8409	/* CS cannot be loaded with NULL in protected mode. */
8410	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8411	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8412	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8413	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8414	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8415	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8416	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8417	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8418	else
8419	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8420
8421	/* SS */
8422	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8423	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8424	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8425	if ( !(pCtx->cr0 & X86_CR0_PE)
8426	\|\| pCtx->cs.Attr.n.u4Type == 3)
8427	{
8428	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8429	}
8430	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8431	{
8432	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8433	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8434	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8435	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8436	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8437	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8438	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8439	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8440	}
8441
8442	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8443	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8444	{
8445	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8446	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8447	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8448	\|\| pCtx->ds.Attr.n.u4Type > 11
8449	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8450	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8451	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8452	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8453	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8454	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8455	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8456	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8457	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8458	}
8459	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8460	{
8461	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8462	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8463	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8464	\|\| pCtx->es.Attr.n.u4Type > 11
8465	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8466	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8467	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8468	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8469	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8470	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8471	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8472	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8473	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8474	}
8475	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8476	{
8477	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8478	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8479	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8480	\|\| pCtx->fs.Attr.n.u4Type > 11
8481	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8482	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8483	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8484	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8485	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8486	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8487	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8488	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8489	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8490	}
8491	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8492	{
8493	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8494	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8495	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8496	\|\| pCtx->gs.Attr.n.u4Type > 11
8497	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8498	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8499	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8500	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8501	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8502	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8503	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8504	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8505	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8506	}
8507	/* 64-bit capable CPUs. */
8508	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8509	if (HMVMX_IS_64BIT_HOST_MODE())
8510	{
8511	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8512	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8513	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8514	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8515	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8516	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8517	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8518	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8519	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8520	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8521	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8522	}
8523	#endif
8524	}
8525	else
8526	{
8527	/* V86 mode checks. */
8528	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8529	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8530	{
8531	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8532	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8533	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8534	}
8535	else
8536	{
8537	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8538	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8539	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8540	}
8541
8542	/* CS */
8543	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8544	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8545	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8546	/* SS */
8547	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8548	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8549	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8550	/* DS */
8551	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8552	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8553	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8554	/* ES */
8555	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8556	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8557	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8558	/* FS */
8559	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8560	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8561	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8562	/* GS */
8563	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8564	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8565	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8566	/* 64-bit capable CPUs. */
8567	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8568	if (HMVMX_IS_64BIT_HOST_MODE())
8569	{
8570	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8571	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8572	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8573	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8574	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8575	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8576	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8577	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8578	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8579	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8580	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8581	}
8582	#endif
8583	}
8584
8585	/*
8586	* TR.
8587	*/
8588	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8589	/* 64-bit capable CPUs. */
8590	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8591	if (HMVMX_IS_64BIT_HOST_MODE())
8592	{
8593	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8594	}
8595	#endif
8596	if (fLongModeGuest)
8597	{
8598	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8599	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8600	}
8601	else
8602	{
8603	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8604	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8605	VMX_IGS_TR_ATTR_TYPE_INVALID);
8606	}
8607	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8608	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8609	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8610	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8611	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8612	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8613	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8614	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8615
8616	/*
8617	* GDTR and IDTR.
8618	*/
8619	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8620	if (HMVMX_IS_64BIT_HOST_MODE())
8621	{
8622	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8623	AssertRCBreak(rc);
8624	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8625
8626	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8627	AssertRCBreak(rc);
8628	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8629	}
8630	#endif
8631
8632	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8633	AssertRCBreak(rc);
8634	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8635
8636	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8637	AssertRCBreak(rc);
8638	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8639
8640	/*
8641	* Guest Non-Register State.
8642	*/
8643	/* Activity State. */
8644	uint32_t u32ActivityState;
8645	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8646	AssertRCBreak(rc);
8647	HMVMX_CHECK_BREAK( !u32ActivityState
8648	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8649	VMX_IGS_ACTIVITY_STATE_INVALID);
8650	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8651	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8652	uint32_t u32IntrState;
8653	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8654	AssertRCBreak(rc);
8655	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8656	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8657	{
8658	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8659	}
8660
8661	/** @todo Activity state and injecting interrupts. Left as a todo since we
8662	* currently don't use activity states but ACTIVE. */
8663
8664	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8665	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8666
8667	/* Guest interruptibility-state. */
8668	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8669	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8670	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8671	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8672	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8673	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8674	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8675	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8676	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8677	if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8678	{
8679	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8680	{
8681	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8682	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8683	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8684	}
8685	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8686	{
8687	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8688	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8689	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8690	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8691	}
8692	}
8693	/** @todo Assumes the processor is not in SMM. */
8694	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8695	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8696	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8697	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8698	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8699	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8700	&& VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8701	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8702	{
8703	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8704	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8705	}
8706
8707	/* Pending debug exceptions. */
8708	if (HMVMX_IS_64BIT_HOST_MODE())
8709	{
8710	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8711	AssertRCBreak(rc);
8712	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8713	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8714	u32Val = u64Val; /* For pending debug exceptions checks below. */
8715	}
8716	else
8717	{
8718	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8719	AssertRCBreak(rc);
8720	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8721	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8722	}
8723
8724	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8725	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8726	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8727	{
8728	if ( (u32Eflags & X86_EFL_TF)
8729	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8730	{
8731	/* Bit 14 is PendingDebug.BS. */
8732	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8733	}
8734	if ( !(u32Eflags & X86_EFL_TF)
8735	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8736	{
8737	/* Bit 14 is PendingDebug.BS. */
8738	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8739	}
8740	}
8741
8742	/* VMCS link pointer. */
8743	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8744	AssertRCBreak(rc);
8745	if (u64Val != UINT64_C(0xffffffffffffffff))
8746	{
8747	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8748	/** @todo Bits beyond the processor's physical-address width MBZ. */
8749	/** @todo 32-bit located in memory referenced by value of this field (as a
8750	* physical address) must contain the processor's VMCS revision ID. */
8751	/** @todo SMM checks. */
8752	}
8753
8754	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8755
8756	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8757	if (uError == VMX_IGS_ERROR)
8758	uError = VMX_IGS_REASON_NOT_FOUND;
8759	} while (0);
8760
8761	pVCpu->hm.s.u32HMError = uError;
8762	return uError;
8763
8764	#undef HMVMX_ERROR_BREAK
8765	#undef HMVMX_CHECK_BREAK
8766	#undef HMVMX_IS_CANONICAL
8767	}
8768
8769	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8770	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8771	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8772
8773	/** @name VM-exit handlers.
8774	* @{
8775	*/
8776
8777	/**
8778	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8779	*/
8780	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8781	{
8782	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8783	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8784	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8785	#if HC_ARCH_BITS == 64
8786	Assert(ASMIntAreEnabled());
8787	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8788	return VINF_SUCCESS;
8789	#endif
8790	return VINF_EM_RAW_INTERRUPT;
8791	}
8792
8793
8794	/**
8795	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8796	*/
8797	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8798	{
8799	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8800	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8801
8802	int rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
8803	AssertRCReturn(rc, rc);
8804
8805	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
8806	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8807	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8808	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
8809
8810	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8811	{
8812	/*
8813	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8814	* anything we inject is not going to cause a VM-exit directly for the event being injected.
8815	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8816	*
8817	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8818	*/
8819	VMXDispatchHostNmi();
8820	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
8821	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8822	return VINF_SUCCESS;
8823	}
8824
8825	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8826	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8827	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8828	{
8829	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8830	return VINF_SUCCESS;
8831	}
8832	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8833	{
8834	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8835	return rc;
8836	}
8837
8838	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
8839	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
8840	switch (uIntType)
8841	{
8842	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8843	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8844	/* no break */
8845	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8846	{
8847	switch (uVector)
8848	{
8849	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8850	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8851	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8852	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8853	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8854	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8855	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8856	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8857	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8858	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8859	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8860	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8861	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8862	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8863	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8864	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8865	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8866	#endif
8867	default:
8868	{
8869	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8870	AssertRCReturn(rc, rc);
8871
8872	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8873	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8874	{
8875	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8876	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8877	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8878
8879	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8880	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
8881	AssertRCReturn(rc, rc);
8882	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
8883	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
8884	0 /* GCPtrFaultAddress */);
8885	AssertRCReturn(rc, rc);
8886	}
8887	else
8888	{
8889	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8890	pVCpu->hm.s.u32HMError = uVector;
8891	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8892	}
8893	break;
8894	}
8895	}
8896	break;
8897	}
8898
8899	default:
8900	{
8901	pVCpu->hm.s.u32HMError = uExitIntInfo;
8902	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
8903	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
8904	break;
8905	}
8906	}
8907	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8908	return rc;
8909	}
8910
8911
8912	/**
8913	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8914	*/
8915	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8916	{
8917	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8918
8919	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8920	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8921	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8922	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8923	AssertRCReturn(rc, rc);
8924
8925	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8926	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8927	return VINF_SUCCESS;
8928	}
8929
8930
8931	/**
8932	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8933	*/
8934	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8935	{
8936	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8937	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8938	HMVMX_RETURN_UNEXPECTED_EXIT();
8939	}
8940
8941
8942	/**
8943	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8944	*/
8945	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8946	{
8947	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8948	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8949	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8950	}
8951
8952
8953	/**
8954	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8955	*/
8956	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8957	{
8958	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8959	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8960	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8961	}
8962
8963
8964	/**
8965	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8966	*/
8967	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8968	{
8969	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8970	PVM pVM = pVCpu->CTX_SUFF(pVM);
8971	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8972	if (RT_LIKELY(rc == VINF_SUCCESS))
8973	{
8974	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8975	Assert(pVmxTransient->cbInstr == 2);
8976	}
8977	else
8978	{
8979	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8980	rc = VERR_EM_INTERPRETER;
8981	}
8982	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8983	return rc;
8984	}
8985
8986
8987	/**
8988	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8989	*/
8990	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8991	{
8992	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8993	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8994	AssertRCReturn(rc, rc);
8995
8996	if (pMixedCtx->cr4 & X86_CR4_SMXE)
8997	return VINF_EM_RAW_EMULATE_INSTR;
8998
8999	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9000	HMVMX_RETURN_UNEXPECTED_EXIT();
9001	}
9002
9003
9004	/**
9005	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9006	*/
9007	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9008	{
9009	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9010	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9011	AssertRCReturn(rc, rc);
9012
9013	PVM pVM = pVCpu->CTX_SUFF(pVM);
9014	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9015	if (RT_LIKELY(rc == VINF_SUCCESS))
9016	{
9017	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9018	Assert(pVmxTransient->cbInstr == 2);
9019	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9020	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9021	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9022	}
9023	else
9024	{
9025	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9026	rc = VERR_EM_INTERPRETER;
9027	}
9028	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9029	return rc;
9030	}
9031
9032
9033	/**
9034	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9035	*/
9036	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9037	{
9038	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9039	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9040	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9041	AssertRCReturn(rc, rc);
9042
9043	PVM pVM = pVCpu->CTX_SUFF(pVM);
9044	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9045	if (RT_LIKELY(rc == VINF_SUCCESS))
9046	{
9047	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9048	Assert(pVmxTransient->cbInstr == 3);
9049	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9050	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9051	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9052	}
9053	else
9054	{
9055	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9056	rc = VERR_EM_INTERPRETER;
9057	}
9058	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9059	return rc;
9060	}
9061
9062
9063	/**
9064	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9065	*/
9066	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9067	{
9068	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9069	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9070	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9071	AssertRCReturn(rc, rc);
9072
9073	PVM pVM = pVCpu->CTX_SUFF(pVM);
9074	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9075	if (RT_LIKELY(rc == VINF_SUCCESS))
9076	{
9077	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9078	Assert(pVmxTransient->cbInstr == 2);
9079	}
9080	else
9081	{
9082	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9083	rc = VERR_EM_INTERPRETER;
9084	}
9085	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9086	return rc;
9087	}
9088
9089
9090	/**
9091	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9092	*/
9093	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9094	{
9095	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9096	PVM pVM = pVCpu->CTX_SUFF(pVM);
9097	Assert(!pVM->hm.s.fNestedPaging);
9098
9099	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9100	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9101	AssertRCReturn(rc, rc);
9102
9103	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9104	rc = VBOXSTRICTRC_VAL(rc2);
9105	if (RT_LIKELY(rc == VINF_SUCCESS))
9106	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9107	else
9108	{
9109	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9110	pVmxTransient->uExitQualification, rc));
9111	}
9112	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9113	return rc;
9114	}
9115
9116
9117	/**
9118	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9119	*/
9120	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9121	{
9122	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9123	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9124	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9125	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9126	AssertRCReturn(rc, rc);
9127
9128	PVM pVM = pVCpu->CTX_SUFF(pVM);
9129	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9130	if (RT_LIKELY(rc == VINF_SUCCESS))
9131	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9132	else
9133	{
9134	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9135	rc = VERR_EM_INTERPRETER;
9136	}
9137	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9138	return rc;
9139	}
9140
9141
9142	/**
9143	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9144	*/
9145	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9146	{
9147	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9148	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9149	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9150	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9151	AssertRCReturn(rc, rc);
9152
9153	PVM pVM = pVCpu->CTX_SUFF(pVM);
9154	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9155	rc = VBOXSTRICTRC_VAL(rc2);
9156	if (RT_LIKELY( rc == VINF_SUCCESS
9157	\|\| rc == VINF_EM_HALT))
9158	{
9159	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9160	AssertRCReturn(rc3, rc3);
9161
9162	if ( rc == VINF_EM_HALT
9163	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9164	{
9165	rc = VINF_SUCCESS;
9166	}
9167	}
9168	else
9169	{
9170	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9171	rc = VERR_EM_INTERPRETER;
9172	}
9173	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9174	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9175	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9176	return rc;
9177	}
9178
9179
9180	/**
9181	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9182	*/
9183	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9184	{
9185	/*
9186	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9187	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9188	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9189	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9190	*/
9191	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9192	HMVMX_RETURN_UNEXPECTED_EXIT();
9193	}
9194
9195
9196	/**
9197	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9198	*/
9199	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9200	{
9201	/*
9202	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9203	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9204	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9205	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9206	*/
9207	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9208	HMVMX_RETURN_UNEXPECTED_EXIT();
9209	}
9210
9211
9212	/**
9213	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9214	*/
9215	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9216	{
9217	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9218	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9219	HMVMX_RETURN_UNEXPECTED_EXIT();
9220	}
9221
9222
9223	/**
9224	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9225	*/
9226	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9227	{
9228	/*
9229	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9230	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9231	* See Intel spec. 25.3 "Other Causes of VM-exits".
9232	*/
9233	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9234	HMVMX_RETURN_UNEXPECTED_EXIT();
9235	}
9236
9237
9238	/**
9239	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9240	* VM-exit.
9241	*/
9242	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9243	{
9244	/*
9245	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9246	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9247	*
9248	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9249	* See Intel spec. "23.8 Restrictions on VMX operation".
9250	*/
9251	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9252	return VINF_SUCCESS;
9253	}
9254
9255
9256	/**
9257	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9258	* VM-exit.
9259	*/
9260	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9261	{
9262	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9263	return VINF_EM_RESET;
9264	}
9265
9266
9267	/**
9268	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9269	*/
9270	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9271	{
9272	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9273	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9274	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9275	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9276	AssertRCReturn(rc, rc);
9277
9278	pMixedCtx->rip++;
9279	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9280	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9281	rc = VINF_SUCCESS;
9282	else
9283	rc = VINF_EM_HALT;
9284
9285	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9286	return rc;
9287	}
9288
9289
9290	/**
9291	* VM-exit handler for instructions that result in a #UD exception delivered to
9292	* the guest.
9293	*/
9294	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9295	{
9296	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9297	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9298	return VINF_SUCCESS;
9299	}
9300
9301
9302	/**
9303	* VM-exit handler for expiry of the VMX preemption timer.
9304	*/
9305	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9306	{
9307	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9308
9309	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9310	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9311
9312	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9313	PVM pVM = pVCpu->CTX_SUFF(pVM);
9314	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9316	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9317	}
9318
9319
9320	/**
9321	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9322	*/
9323	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9324	{
9325	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9326
9327	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9328	/** @todo check if XSETBV is supported by the recompiler. */
9329	return VERR_EM_INTERPRETER;
9330	}
9331
9332
9333	/**
9334	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9335	*/
9336	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9337	{
9338	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9339
9340	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9341	/** @todo implement EMInterpretInvpcid() */
9342	return VERR_EM_INTERPRETER;
9343	}
9344
9345
9346	/**
9347	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9348	* Error VM-exit.
9349	*/
9350	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9351	{
9352	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9353	AssertRCReturn(rc, rc);
9354
9355	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9356	NOREF(uInvalidReason);
9357
9358	#ifdef VBOX_STRICT
9359	uint32_t uIntrState;
9360	HMVMXHCUINTREG uHCReg;
9361	uint64_t u64Val;
9362	uint32_t u32Val;
9363
9364	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9365	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9366	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9367	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9368	AssertRCReturn(rc, rc);
9369
9370	Log4(("uInvalidReason %u\n", uInvalidReason));
9371	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9372	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9373	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9374	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9375
9376	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9377	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9378	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9379	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9380	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9381	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9382	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9383	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9384	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9385	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9386	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9387	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9388	#endif
9389
9390	PVM pVM = pVCpu->CTX_SUFF(pVM);
9391	HMDumpRegs(pVM, pVCpu, pMixedCtx);
9392
9393	return VERR_VMX_INVALID_GUEST_STATE;
9394	}
9395
9396
9397	/**
9398	* VM-exit handler for VM-entry failure due to an MSR-load
9399	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9400	*/
9401	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9402	{
9403	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9404	HMVMX_RETURN_UNEXPECTED_EXIT();
9405	}
9406
9407
9408	/**
9409	* VM-exit handler for VM-entry failure due to a machine-check event
9410	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9411	*/
9412	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9413	{
9414	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9415	HMVMX_RETURN_UNEXPECTED_EXIT();
9416	}
9417
9418
9419	/**
9420	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9421	* theory.
9422	*/
9423	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9424	{
9425	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9426	return VERR_VMX_UNDEFINED_EXIT_CODE;
9427	}
9428
9429
9430	/**
9431	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9432	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9433	* Conditional VM-exit.
9434	*/
9435	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9436	{
9437	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9438
9439	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9440	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9441	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9442	return VERR_EM_INTERPRETER;
9443	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9444	HMVMX_RETURN_UNEXPECTED_EXIT();
9445	}
9446
9447
9448	/**
9449	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9450	*/
9451	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9452	{
9453	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9454
9455	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9456	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9457	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9458	return VERR_EM_INTERPRETER;
9459	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9460	HMVMX_RETURN_UNEXPECTED_EXIT();
9461	}
9462
9463
9464	/**
9465	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9466	*/
9467	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9468	{
9469	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9470
9471	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9472	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9473	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9474	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9475	AssertRCReturn(rc, rc);
9476	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9477
9478	PVM pVM = pVCpu->CTX_SUFF(pVM);
9479	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9480	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9481	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9482	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9483
9484	if (RT_LIKELY(rc == VINF_SUCCESS))
9485	{
9486	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9487	Assert(pVmxTransient->cbInstr == 2);
9488	}
9489	return rc;
9490	}
9491
9492
9493	/**
9494	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9495	*/
9496	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9497	{
9498	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9499	PVM pVM = pVCpu->CTX_SUFF(pVM);
9500	int rc = VINF_SUCCESS;
9501
9502	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9503	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9504	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9505	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9506	AssertRCReturn(rc, rc);
9507	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9508
9509	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9510	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9511	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9512
9513	if (RT_LIKELY(rc == VINF_SUCCESS))
9514	{
9515	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9516
9517	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9518	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9519	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9520	{
9521	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9522	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9523	EMInterpretWrmsr() changes it. */
9524	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9525	}
9526	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9527	{
9528	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9529	AssertRCReturn(rc, rc);
9530	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
9531	}
9532	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9533	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9534
9535	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9536	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9537	{
9538	switch (pMixedCtx->ecx)
9539	{
9540	case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9541	case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9542	case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9543	case MSR_K8_FS_BASE: /* no break */
9544	case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9545	case MSR_K8_KERNEL_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS); break;
9546	}
9547	}
9548	#ifdef VBOX_STRICT
9549	else
9550	{
9551	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9552	switch (pMixedCtx->ecx)
9553	{
9554	case MSR_IA32_SYSENTER_CS:
9555	case MSR_IA32_SYSENTER_EIP:
9556	case MSR_IA32_SYSENTER_ESP:
9557	case MSR_K8_FS_BASE:
9558	case MSR_K8_GS_BASE:
9559	{
9560	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9561	HMVMX_RETURN_UNEXPECTED_EXIT();
9562	}
9563
9564	case MSR_K8_LSTAR:
9565	case MSR_K6_STAR:
9566	case MSR_K8_SF_MASK:
9567	case MSR_K8_TSC_AUX:
9568	case MSR_K8_KERNEL_GS_BASE:
9569	{
9570	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9571	pMixedCtx->ecx));
9572	HMVMX_RETURN_UNEXPECTED_EXIT();
9573	}
9574	}
9575	}
9576	#endif /* VBOX_STRICT */
9577	}
9578	return rc;
9579	}
9580
9581
9582	/**
9583	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9584	*/
9585	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9586	{
9587	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9588
9589	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9590	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9591	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9592	return VERR_EM_INTERPRETER;
9593	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9594	HMVMX_RETURN_UNEXPECTED_EXIT();
9595	}
9596
9597
9598	/**
9599	* VM-exit handler for when the TPR value is lowered below the specified
9600	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9601	*/
9602	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9603	{
9604	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9605	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9606
9607	/*
9608	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9609	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9610	* resume guest execution.
9611	*/
9612	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9613	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9614	return VINF_SUCCESS;
9615	}
9616
9617
9618	/**
9619	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9620	* VM-exit.
9621	*
9622	* @retval VINF_SUCCESS when guest execution can continue.
9623	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9624	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9625	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9626	* recompiler.
9627	*/
9628	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9629	{
9630	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9631	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9632	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9633	AssertRCReturn(rc, rc);
9634
9635	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9636	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9637	PVM pVM = pVCpu->CTX_SUFF(pVM);
9638	switch (uAccessType)
9639	{
9640	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9641	{
9642	#if 0
9643	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9644	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9645	#else
9646	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9647	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9648	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9649	#endif
9650	AssertRCReturn(rc, rc);
9651
9652	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9653	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9654	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9655	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
9656
9657	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9658	{
9659	case 0: /* CR0 */
9660	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9661	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9662	break;
9663	case 2: /* CR2 */
9664	/* Nothing to do here, CR2 it's not part of the VMCS. */
9665	break;
9666	case 3: /* CR3 */
9667	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9668	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
9669	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9670	break;
9671	case 4: /* CR4 */
9672	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9673	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9674	break;
9675	case 8: /* CR8 */
9676	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9677	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9678	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9679	break;
9680	default:
9681	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9682	break;
9683	}
9684
9685	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9686	break;
9687	}
9688
9689	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9690	{
9691	/* EMInterpretCRxRead() requires EFER MSR, CS. */
9692	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9693	AssertRCReturn(rc, rc);
9694	Assert( !pVM->hm.s.fNestedPaging
9695	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9696	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9697
9698	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9699	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9700	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9701
9702	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9703	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9704	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9705	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9706	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9707	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9708	break;
9709	}
9710
9711	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9712	{
9713	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9714	AssertRCReturn(rc, rc);
9715	rc = EMInterpretCLTS(pVM, pVCpu);
9716	AssertRCReturn(rc, rc);
9717	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9718	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9719	Log4(("CRX CLTS write rc=%d\n", rc));
9720	break;
9721	}
9722
9723	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9724	{
9725	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9726	AssertRCReturn(rc, rc);
9727	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9728	if (RT_LIKELY(rc == VINF_SUCCESS))
9729	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9730	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9731	Log4(("CRX LMSW write rc=%d\n", rc));
9732	break;
9733	}
9734
9735	default:
9736	{
9737	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9738	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9739	}
9740	}
9741
9742	/* Validate possible error codes. */
9743	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
9744	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9745	if (RT_SUCCESS(rc))
9746	{
9747	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9748	AssertRCReturn(rc2, rc2);
9749	}
9750
9751	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9752	return rc;
9753	}
9754
9755
9756	/**
9757	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9758	* VM-exit.
9759	*/
9760	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9761	{
9762	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9763	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9764
9765	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9766	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9767	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9768	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9769	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9770	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9771	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9772	AssertRCReturn(rc2, rc2);
9773
9774	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9775	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9776	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9777	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9778	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9779	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9780	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9781
9782	/* I/O operation lookup arrays. */
9783	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9784	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9785
9786	VBOXSTRICTRC rcStrict;
9787	const uint32_t cbValue = s_aIOSizes[uIOWidth];
9788	const uint32_t cbInstr = pVmxTransient->cbInstr;
9789	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9790	PVM pVM = pVCpu->CTX_SUFF(pVM);
9791	if (fIOString)
9792	{
9793	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158}*/
9794	/*
9795	* INS/OUTS - I/O String instruction.
9796	*
9797	* Use instruction-information if available, otherwise fall back on
9798	* interpreting the instruction.
9799	*/
9800	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9801	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9802	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9803	{
9804	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
9805	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9806	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9807	AssertRCReturn(rc2, rc2);
9808	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9809	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9810	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9811	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9812	if (fIOWrite)
9813	{
9814	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9815	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9816	}
9817	else
9818	{
9819	/*
9820	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
9821	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
9822	* See Intel Instruction spec. for "INS".
9823	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
9824	*/
9825	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9826	}
9827	}
9828	else
9829	{
9830	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9831	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9832	AssertRCReturn(rc2, rc2);
9833	rcStrict = IEMExecOne(pVCpu);
9834	}
9835	/** @todo IEM needs to be setting these flags somehow. */
9836	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9837	fUpdateRipAlready = true;
9838	#else
9839	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9840	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9841	if (RT_SUCCESS(rcStrict))
9842	{
9843	if (fIOWrite)
9844	{
9845	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9846	(DISCPUMODE)pDis->uAddrMode, cbValue);
9847	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9848	}
9849	else
9850	{
9851	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9852	(DISCPUMODE)pDis->uAddrMode, cbValue);
9853	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9854	}
9855	}
9856	else
9857	{
9858	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9859	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9860	}
9861	#endif
9862	}
9863	else
9864	{
9865	/*
9866	* IN/OUT - I/O instruction.
9867	*/
9868	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9869	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9870	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9871	if (fIOWrite)
9872	{
9873	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9874	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9875	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9876	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9877	}
9878	else
9879	{
9880	uint32_t u32Result = 0;
9881	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9882	if (IOM_SUCCESS(rcStrict))
9883	{
9884	/* Save result of I/O IN instr. in AL/AX/EAX. */
9885	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
9886	}
9887	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9888	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9889	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9890	}
9891	}
9892
9893	if (IOM_SUCCESS(rcStrict))
9894	{
9895	if (!fUpdateRipAlready)
9896	{
9897	pMixedCtx->rip += cbInstr;
9898	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9899	}
9900
9901	/*
9902	* If any I/O breakpoints are armed, we need to check if one triggered
9903	* and take appropriate action.
9904	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9905	*/
9906	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9907	AssertRCReturn(rc2, rc2);
9908
9909	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9910	* execution engines about whether hyper BPs and such are pending. */
9911	uint32_t const uDr7 = pMixedCtx->dr[7];
9912	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9913	&& X86_DR7_ANY_RW_IO(uDr7)
9914	&& (pMixedCtx->cr4 & X86_CR4_DE))
9915	\|\| DBGFBpIsHwIoArmed(pVM)))
9916	{
9917	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9918
9919	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
9920	VMMRZCallRing3Disable(pVCpu);
9921	HM_DISABLE_PREEMPT_IF_NEEDED();
9922
9923	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
9924
9925	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9926	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9927	{
9928	/* Raise #DB. */
9929	if (fIsGuestDbgActive)
9930	ASMSetDR6(pMixedCtx->dr[6]);
9931	if (pMixedCtx->dr[7] != uDr7)
9932	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
9933
9934	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9935	}
9936	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9937	else if ( rcStrict2 != VINF_SUCCESS
9938	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
9939	rcStrict = rcStrict2;
9940
9941	HM_RESTORE_PREEMPT_IF_NEEDED();
9942	VMMRZCallRing3Enable(pVCpu);
9943	}
9944	}
9945
9946	#ifdef DEBUG
9947	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9948	Assert(!fIOWrite);
9949	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9950	Assert(fIOWrite);
9951	else
9952	{
9953	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9954	* statuses, that the VMM device and some others may return. See
9955	* IOM_SUCCESS() for guidance. */
9956	AssertMsg( RT_FAILURE(rcStrict)
9957	\|\| rcStrict == VINF_SUCCESS
9958	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
9959	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
9960	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
9961	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9962	}
9963	#endif
9964
9965	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9966	return VBOXSTRICTRC_TODO(rcStrict);
9967	}
9968
9969
9970	/**
9971	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9972	* VM-exit.
9973	*/
9974	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9975	{
9976	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9977
9978	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
9979	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9980	AssertRCReturn(rc, rc);
9981	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9982	{
9983	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9984	AssertRCReturn(rc, rc);
9985	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9986	{
9987	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9988
9989	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9990	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9991	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9992	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9993	{
9994	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9995	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9996
9997	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9998	Assert(!pVCpu->hm.s.Event.fPending);
9999	pVCpu->hm.s.Event.fPending = true;
10000	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10001	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10002	AssertRCReturn(rc, rc);
10003	if (fErrorCodeValid)
10004	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10005	else
10006	pVCpu->hm.s.Event.u32ErrCode = 0;
10007	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10008	&& uVector == X86_XCPT_PF)
10009	{
10010	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10011	}
10012
10013	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10014	}
10015	}
10016	}
10017
10018	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10019	* emulation. */
10020	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10021	return VERR_EM_INTERPRETER;
10022	}
10023
10024
10025	/**
10026	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10027	*/
10028	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10029	{
10030	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10031	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10032	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10033	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10034	AssertRCReturn(rc, rc);
10035	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10036	return VINF_EM_DBG_STEPPED;
10037	}
10038
10039
10040	/**
10041	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10042	*/
10043	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10044	{
10045	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10046
10047	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10048	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10049	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10050	return VINF_SUCCESS;
10051	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10052	return rc;
10053
10054	#if 0
10055	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10056	* just sync the whole thing. */
10057	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10058	#else
10059	/* Aggressive state sync. for now. */
10060	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10061	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10062	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10063	#endif
10064	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10065	AssertRCReturn(rc, rc);
10066
10067	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10068	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10069	switch (uAccessType)
10070	{
10071	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10072	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10073	{
10074	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10075	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10076	{
10077	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10078	}
10079
10080	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10081	GCPhys &= PAGE_BASE_GC_MASK;
10082	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10083	PVM pVM = pVCpu->CTX_SUFF(pVM);
10084	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10085	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10086
10087	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10088	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10089	CPUMCTX2CORE(pMixedCtx), GCPhys);
10090	rc = VBOXSTRICTRC_VAL(rc2);
10091	Log4(("ApicAccess rc=%d\n", rc));
10092	if ( rc == VINF_SUCCESS
10093	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10094	\|\| rc == VERR_PAGE_NOT_PRESENT)
10095	{
10096	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10097	\| HM_CHANGED_GUEST_RSP
10098	\| HM_CHANGED_GUEST_RFLAGS
10099	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10100	rc = VINF_SUCCESS;
10101	}
10102	break;
10103	}
10104
10105	default:
10106	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10107	rc = VINF_EM_RAW_EMULATE_INSTR;
10108	break;
10109	}
10110
10111	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10112	return rc;
10113	}
10114
10115
10116	/**
10117	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10118	* VM-exit.
10119	*/
10120	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10121	{
10122	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10123
10124	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10125	if (pVmxTransient->fWasGuestDebugStateActive)
10126	{
10127	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10128	HMVMX_RETURN_UNEXPECTED_EXIT();
10129	}
10130
10131	int rc = VERR_INTERNAL_ERROR_5;
10132	if ( !DBGFIsStepping(pVCpu)
10133	&& !pVCpu->hm.s.fSingleInstruction
10134	&& !pVmxTransient->fWasHyperDebugStateActive)
10135	{
10136	/* Don't intercept MOV DRx and #DB any more. */
10137	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10138	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10139	AssertRCReturn(rc, rc);
10140
10141	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10142	{
10143	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10144	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10145	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10146	AssertRCReturn(rc, rc);
10147	#endif
10148	}
10149
10150	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10151	VMMRZCallRing3Disable(pVCpu);
10152	HM_DISABLE_PREEMPT_IF_NEEDED();
10153
10154	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10155	PVM pVM = pVCpu->CTX_SUFF(pVM);
10156	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10157	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10158
10159	HM_RESTORE_PREEMPT_IF_NEEDED();
10160	VMMRZCallRing3Enable(pVCpu);
10161
10162	#ifdef VBOX_WITH_STATISTICS
10163	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10164	AssertRCReturn(rc, rc);
10165	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10166	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10167	else
10168	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10169	#endif
10170	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10171	return VINF_SUCCESS;
10172	}
10173
10174	/*
10175	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10176	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10177	*/
10178	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10179	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10180	AssertRCReturn(rc, rc);
10181	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10182
10183	PVM pVM = pVCpu->CTX_SUFF(pVM);
10184	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10185	{
10186	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10187	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10188	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10189	if (RT_SUCCESS(rc))
10190	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10191	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10192	}
10193	else
10194	{
10195	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10196	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10197	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10198	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10199	}
10200
10201	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10202	if (RT_SUCCESS(rc))
10203	{
10204	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10205	AssertRCReturn(rc2, rc2);
10206	}
10207	return rc;
10208	}
10209
10210
10211	/**
10212	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10213	* Conditional VM-exit.
10214	*/
10215	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10216	{
10217	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10218	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10219
10220	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10221	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10222	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10223	return VINF_SUCCESS;
10224	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10225	return rc;
10226
10227	RTGCPHYS GCPhys = 0;
10228	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10229
10230	#if 0
10231	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10232	#else
10233	/* Aggressive state sync. for now. */
10234	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10235	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10236	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10237	#endif
10238	AssertRCReturn(rc, rc);
10239
10240	/*
10241	* If we succeed, resume guest execution.
10242	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10243	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10244	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10245	* weird case. See @bugref{6043}.
10246	*/
10247	PVM pVM = pVCpu->CTX_SUFF(pVM);
10248	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10249	rc = VBOXSTRICTRC_VAL(rc2);
10250	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10251	if ( rc == VINF_SUCCESS
10252	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10253	\|\| rc == VERR_PAGE_NOT_PRESENT)
10254	{
10255	/* Successfully handled MMIO operation. */
10256	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10257	\| HM_CHANGED_GUEST_RSP
10258	\| HM_CHANGED_GUEST_RFLAGS
10259	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10260	rc = VINF_SUCCESS;
10261	}
10262	return rc;
10263	}
10264
10265
10266	/**
10267	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10268	* VM-exit.
10269	*/
10270	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10271	{
10272	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10273	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10274
10275	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10276	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10277	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10278	return VINF_SUCCESS;
10279	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10280	return rc;
10281
10282	RTGCPHYS GCPhys = 0;
10283	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10284	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10285	#if 0
10286	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10287	#else
10288	/* Aggressive state sync. for now. */
10289	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10290	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10291	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10292	#endif
10293	AssertRCReturn(rc, rc);
10294
10295	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10296	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10297
10298	RTGCUINT uErrorCode = 0;
10299	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10300	uErrorCode \|= X86_TRAP_PF_ID;
10301	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10302	uErrorCode \|= X86_TRAP_PF_RW;
10303	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10304	uErrorCode \|= X86_TRAP_PF_P;
10305
10306	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10307
10308	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10309	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10310
10311	/* Handle the pagefault trap for the nested shadow table. */
10312	PVM pVM = pVCpu->CTX_SUFF(pVM);
10313	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10314	TRPMResetTrap(pVCpu);
10315
10316	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10317	if ( rc == VINF_SUCCESS
10318	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10319	\|\| rc == VERR_PAGE_NOT_PRESENT)
10320	{
10321	/* Successfully synced our nested page tables. */
10322	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10323	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10324	\| HM_CHANGED_GUEST_RSP
10325	\| HM_CHANGED_GUEST_RFLAGS);
10326	return VINF_SUCCESS;
10327	}
10328
10329	Log4(("EPT return to ring-3 rc=%d\n"));
10330	return rc;
10331	}
10332
10333	/** @} */
10334
10335	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10336	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10337	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10338
10339	/** @name VM-exit exception handlers.
10340	* @{
10341	*/
10342
10343	/**
10344	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10345	*/
10346	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10347	{
10348	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10349	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10350
10351	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10352	AssertRCReturn(rc, rc);
10353
10354	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10355	{
10356	/* Old-style FPU error reporting needs some extra work. */
10357	/** @todo don't fall back to the recompiler, but do it manually. */
10358	return VERR_EM_INTERPRETER;
10359	}
10360
10361	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10362	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10363	return rc;
10364	}
10365
10366
10367	/**
10368	* VM-exit exception handler for #BP (Breakpoint exception).
10369	*/
10370	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10371	{
10372	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10373	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10374
10375	/** @todo Try optimize this by not saving the entire guest state unless
10376	* really needed. */
10377	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10378	AssertRCReturn(rc, rc);
10379
10380	PVM pVM = pVCpu->CTX_SUFF(pVM);
10381	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10382	if (rc == VINF_EM_RAW_GUEST_TRAP)
10383	{
10384	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10385	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10386	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10387	AssertRCReturn(rc, rc);
10388
10389	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10390	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10391	}
10392
10393	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10394	return rc;
10395	}
10396
10397
10398	/**
10399	* VM-exit exception handler for #DB (Debug exception).
10400	*/
10401	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10402	{
10403	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10404	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10405	Log6(("XcptDB\n"));
10406
10407	/*
10408	* Get the DR6-like values from the exit qualification and pass it to DBGF
10409	* for processing.
10410	*/
10411	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10412	AssertRCReturn(rc, rc);
10413
10414	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10415	uint64_t uDR6 = X86_DR6_INIT_VAL;
10416	uDR6 \|= ( pVmxTransient->uExitQualification
10417	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10418
10419	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10420	if (rc == VINF_EM_RAW_GUEST_TRAP)
10421	{
10422	/*
10423	* The exception was for the guest. Update DR6, DR7.GD and
10424	* IA32_DEBUGCTL.LBR before forwarding it.
10425	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10426	*/
10427	VMMRZCallRing3Disable(pVCpu);
10428	HM_DISABLE_PREEMPT_IF_NEEDED();
10429
10430	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10431	pMixedCtx->dr[6] \|= uDR6;
10432	if (CPUMIsGuestDebugStateActive(pVCpu))
10433	ASMSetDR6(pMixedCtx->dr[6]);
10434
10435	HM_RESTORE_PREEMPT_IF_NEEDED();
10436	VMMRZCallRing3Enable(pVCpu);
10437
10438	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10439	AssertRCReturn(rc, rc);
10440
10441	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10442	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10443
10444	/* Paranoia. */
10445	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10446	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10447
10448	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10449	AssertRCReturn(rc, rc);
10450
10451	/*
10452	* Raise #DB in the guest.
10453	*/
10454	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10455	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10456	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10457	AssertRCReturn(rc, rc);
10458	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10459	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10460	return VINF_SUCCESS;
10461	}
10462
10463	/*
10464	* Not a guest trap, must be a hypervisor related debug event then.
10465	* Update DR6 in case someone is interested in it.
10466	*/
10467	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10468	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10469	CPUMSetHyperDR6(pVCpu, uDR6);
10470
10471	return rc;
10472	}
10473
10474
10475	/**
10476	* VM-exit exception handler for #NM (Device-not-available exception: floating
10477	* point exception).
10478	*/
10479	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10480	{
10481	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10482
10483	/* We require CR0 and EFER. EFER is always up-to-date. */
10484	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10485	AssertRCReturn(rc, rc);
10486
10487	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10488	VMMRZCallRing3Disable(pVCpu);
10489	HM_DISABLE_PREEMPT_IF_NEEDED();
10490
10491	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10492	if (pVmxTransient->fWasGuestFPUStateActive)
10493	{
10494	rc = VINF_EM_RAW_GUEST_TRAP;
10495	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10496	}
10497	else
10498	{
10499	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10500	Assert(!pVmxTransient->fWasGuestFPUStateActive);
10501	#endif
10502	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10503	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10504	}
10505
10506	HM_RESTORE_PREEMPT_IF_NEEDED();
10507	VMMRZCallRing3Enable(pVCpu);
10508
10509	if (rc == VINF_SUCCESS)
10510	{
10511	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
10512	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10513	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10514	}
10515	else
10516	{
10517	/* Forward #NM to the guest. */
10518	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10519	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10520	AssertRCReturn(rc, rc);
10521	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10522	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
10523	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10524	}
10525
10526	return VINF_SUCCESS;
10527	}
10528
10529
10530	/**
10531	* VM-exit exception handler for #GP (General-protection exception).
10532	*
10533	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
10534	*/
10535	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10536	{
10537	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10538	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10539
10540	int rc = VERR_INTERNAL_ERROR_5;
10541	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10542	{
10543	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10544	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10545	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10546	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10547	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10548	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10549	AssertRCReturn(rc, rc);
10550	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
10551	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10552	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10553	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10554	return rc;
10555	#else
10556	/* We don't intercept #GP. */
10557	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10558	return VERR_VMX_UNEXPECTED_EXCEPTION;
10559	#endif
10560	}
10561
10562	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10563	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10564
10565	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10566	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10567	AssertRCReturn(rc, rc);
10568
10569	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10570	uint32_t cbOp = 0;
10571	PVM pVM = pVCpu->CTX_SUFF(pVM);
10572	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10573	if (RT_SUCCESS(rc))
10574	{
10575	rc = VINF_SUCCESS;
10576	Assert(cbOp == pDis->cbInstr);
10577	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10578	switch (pDis->pCurInstr->uOpcode)
10579	{
10580	case OP_CLI:
10581	{
10582	pMixedCtx->eflags.Bits.u1IF = 0;
10583	pMixedCtx->rip += pDis->cbInstr;
10584	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10585	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10586	break;
10587	}
10588
10589	case OP_STI:
10590	{
10591	pMixedCtx->eflags.Bits.u1IF = 1;
10592	pMixedCtx->rip += pDis->cbInstr;
10593	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10594	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10595	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10596	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10597	break;
10598	}
10599
10600	case OP_HLT:
10601	{
10602	rc = VINF_EM_HALT;
10603	pMixedCtx->rip += pDis->cbInstr;
10604	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10605	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10606	break;
10607	}
10608
10609	case OP_POPF:
10610	{
10611	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10612	uint32_t cbParm = 0;
10613	uint32_t uMask = 0;
10614	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10615	{
10616	cbParm = 4;
10617	uMask = 0xffffffff;
10618	}
10619	else
10620	{
10621	cbParm = 2;
10622	uMask = 0xffff;
10623	}
10624
10625	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
10626	RTGCPTR GCPtrStack = 0;
10627	X86EFLAGS Eflags;
10628	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10629	&GCPtrStack);
10630	if (RT_SUCCESS(rc))
10631	{
10632	Assert(sizeof(Eflags.u32) >= cbParm);
10633	Eflags.u32 = 0;
10634	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10635	}
10636	if (RT_FAILURE(rc))
10637	{
10638	rc = VERR_EM_INTERPRETER;
10639	break;
10640	}
10641	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10642	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10643	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10644	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
10645	pMixedCtx->esp += cbParm;
10646	pMixedCtx->esp &= uMask;
10647	pMixedCtx->rip += pDis->cbInstr;
10648
10649	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10650	\| HM_CHANGED_GUEST_RSP
10651	\| HM_CHANGED_GUEST_RFLAGS);
10652	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10653	break;
10654	}
10655
10656	case OP_PUSHF:
10657	{
10658	uint32_t cbParm = 0;
10659	uint32_t uMask = 0;
10660	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10661	{
10662	cbParm = 4;
10663	uMask = 0xffffffff;
10664	}
10665	else
10666	{
10667	cbParm = 2;
10668	uMask = 0xffff;
10669	}
10670
10671	/* Get the stack pointer & push the contents of eflags onto the stack. */
10672	RTGCPTR GCPtrStack = 0;
10673	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10674	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10675	if (RT_FAILURE(rc))
10676	{
10677	rc = VERR_EM_INTERPRETER;
10678	break;
10679	}
10680	X86EFLAGS Eflags = pMixedCtx->eflags;
10681	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10682	Eflags.Bits.u1RF = 0;
10683	Eflags.Bits.u1VM = 0;
10684
10685	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10686	if (RT_FAILURE(rc))
10687	{
10688	rc = VERR_EM_INTERPRETER;
10689	break;
10690	}
10691	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10692	pMixedCtx->esp -= cbParm;
10693	pMixedCtx->esp &= uMask;
10694	pMixedCtx->rip += pDis->cbInstr;
10695	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
10696	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10697	break;
10698	}
10699
10700	case OP_IRET:
10701	{
10702	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10703	* instruction reference. */
10704	RTGCPTR GCPtrStack = 0;
10705	uint32_t uMask = 0xffff;
10706	uint16_t aIretFrame[3];
10707	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
10708	{
10709	rc = VERR_EM_INTERPRETER;
10710	break;
10711	}
10712	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10713	&GCPtrStack);
10714	if (RT_SUCCESS(rc))
10715	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10716	if (RT_FAILURE(rc))
10717	{
10718	rc = VERR_EM_INTERPRETER;
10719	break;
10720	}
10721	pMixedCtx->eip = 0;
10722	pMixedCtx->ip = aIretFrame[0];
10723	pMixedCtx->cs.Sel = aIretFrame[1];
10724	pMixedCtx->cs.ValidSel = aIretFrame[1];
10725	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10726	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10727	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10728	pMixedCtx->sp += sizeof(aIretFrame);
10729	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10730	\| HM_CHANGED_GUEST_SEGMENT_REGS
10731	\| HM_CHANGED_GUEST_RSP
10732	\| HM_CHANGED_GUEST_RFLAGS);
10733	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10734	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10735	break;
10736	}
10737
10738	case OP_INT:
10739	{
10740	uint16_t uVector = pDis->Param1.uValue & 0xff;
10741	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10742	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10743	break;
10744	}
10745
10746	case OP_INTO:
10747	{
10748	if (pMixedCtx->eflags.Bits.u1OF)
10749	{
10750	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10751	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10752	}
10753	break;
10754	}
10755
10756	default:
10757	{
10758	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10759	EMCODETYPE_SUPERVISOR);
10760	rc = VBOXSTRICTRC_VAL(rc2);
10761	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
10762	Log4(("#GP rc=%Rrc\n", rc));
10763	break;
10764	}
10765	}
10766	}
10767	else
10768	rc = VERR_EM_INTERPRETER;
10769
10770	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
10771	("#GP Unexpected rc=%Rrc\n", rc));
10772	return rc;
10773	}
10774
10775
10776	/**
10777	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10778	* the exception reported in the VMX transient structure back into the VM.
10779	*
10780	* @remarks Requires uExitIntInfo in the VMX transient structure to be
10781	* up-to-date.
10782	*/
10783	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10784	{
10785	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10786
10787	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10788	hmR0VmxCheckExitDueToEventDelivery(). */
10789	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10790	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10791	AssertRCReturn(rc, rc);
10792	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10793
10794	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10795	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10796	return VINF_SUCCESS;
10797	}
10798
10799
10800	/**
10801	* VM-exit exception handler for #PF (Page-fault exception).
10802	*/
10803	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10804	{
10805	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10806	PVM pVM = pVCpu->CTX_SUFF(pVM);
10807	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10808	rc \|= hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10809	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10810	AssertRCReturn(rc, rc);
10811
10812	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
10813	if (pVM->hm.s.fNestedPaging)
10814	{
10815	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10816	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10817	{
10818	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10819	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10820	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
10821	}
10822	else
10823	{
10824	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10825	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10826	Log4(("Pending #DF due to vectoring #PF. NP\n"));
10827	}
10828	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10829	return rc;
10830	}
10831	#else
10832	Assert(!pVM->hm.s.fNestedPaging);
10833	#endif
10834
10835	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10836	AssertRCReturn(rc, rc);
10837
10838	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10839	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
10840
10841	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
10842	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
10843	(RTGCPTR)pVmxTransient->uExitQualification);
10844
10845	Log4(("#PF: rc=%Rrc\n", rc));
10846	if (rc == VINF_SUCCESS)
10847	{
10848	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10849	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
10850	* memory? We don't update the whole state here... */
10851	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10852	\| HM_CHANGED_GUEST_RSP
10853	\| HM_CHANGED_GUEST_RFLAGS
10854	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10855	TRPMResetTrap(pVCpu);
10856	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10857	return rc;
10858	}
10859	else if (rc == VINF_EM_RAW_GUEST_TRAP)
10860	{
10861	if (!pVmxTransient->fVectoringPF)
10862	{
10863	/* It's a guest page fault and needs to be reflected to the guest. */
10864	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10865	TRPMResetTrap(pVCpu);
10866	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10867	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10868	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10869	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10870	}
10871	else
10872	{
10873	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10874	TRPMResetTrap(pVCpu);
10875	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10876	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10877	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10878	}
10879
10880	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10881	return VINF_SUCCESS;
10882	}
10883
10884	TRPMResetTrap(pVCpu);
10885	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10886	return rc;
10887	}
10888
10889	/** @} */
10890

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49159

Download in other formats: