HMVMXR0.cpp@ 49207

Last change on this file since 49207 was 49207, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: Fix MSR_TSC_AUX, use the right check to be doubly sure.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 453.8 KB

Line
1	/* $Id: HMVMXR0.cpp 49207 2013-10-21 10:20:13Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	#define HMVMX_SAVE_FULL_GUEST_STATE
40	#define HMVMX_SYNC_FULL_GUEST_STATE
41	#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	#define HMVMX_ALWAYS_TRAP_PF
44	#define HMVMX_ALWAYS_SWAP_FPU_STATE
45	#endif
46
47
48	/*******************************************************************************
49	* Defined Constants And Macros *
50	*******************************************************************************/
51	#if defined(RT_ARCH_AMD64)
52	# define HMVMX_IS_64BIT_HOST_MODE() (true)
53	typedef RTHCUINTREG HMVMXHCUINTREG;
54	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
55	extern "C" uint32_t g_fVMXIs64bitHost;
56	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
57	typedef uint64_t HMVMXHCUINTREG;
58	#else
59	# define HMVMX_IS_64BIT_HOST_MODE() (false)
60	typedef RTHCUINTREG HMVMXHCUINTREG;
61	#endif
62
63	/** Use the function table. */
64	#define HMVMX_USE_FUNCTION_TABLE
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127	/** @} */
128
129	/** @name
130	* States of the VMCS.
131	*
132	* This does not reflect all possible VMCS states but currently only those
133	* needed for maintaining the VMCS consistently even when thread-context hooks
134	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
135	*/
136	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139	/** @} */
140
141	/**
142	* Exception bitmap mask for real-mode guests (real-on-v86).
143	*
144	* We need to intercept all exceptions manually (except #PF). #NM is also
145	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146	* even in real-mode if we have Nested Paging support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*******************************************************************************
197	* Structures and Typedefs *
198	*******************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG uEflags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
213	uint64_t u64LStarMsr;
214	/** The guest's TPR value used for TPR shadowing. */
215	uint8_t u8GuestTpr;
216	/** Alignment. */
217	uint8_t abAlignment0[7];
218
219	/** The basic VM-exit reason. */
220	uint16_t uExitReason;
221	/** Alignment. */
222	uint16_t u16Alignment0;
223	/** The VM-exit interruption error code. */
224	uint32_t uExitIntErrorCode;
225	/** The VM-exit exit qualification. */
226	uint64_t uExitQualification;
227
228	/** The VM-exit interruption-information field. */
229	uint32_t uExitIntInfo;
230	/** The VM-exit instruction-length field. */
231	uint32_t cbInstr;
232	/** The VM-exit instruction-information field. */
233	union
234	{
235	/** Plain unsigned int representation. */
236	uint32_t u;
237	/** INS and OUTS information. */
238	struct
239	{
240	uint32_t u6Reserved0 : 7;
241	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
242	uint32_t u3AddrSize : 3;
243	uint32_t u5Reserved1 : 5;
244	/** The segment register (X86_SREG_XXX). */
245	uint32_t iSegReg : 3;
246	uint32_t uReserved2 : 14;
247	} StrIo;
248	} ExitInstrInfo;
249	/** Whether the VM-entry failed or not. */
250	bool fVMEntryFailed;
251	/** Alignment. */
252	uint8_t abAlignment1[3];
253
254	/** The VM-entry interruption-information field. */
255	uint32_t uEntryIntInfo;
256	/** The VM-entry exception error code field. */
257	uint32_t uEntryXcptErrorCode;
258	/** The VM-entry instruction length field. */
259	uint32_t cbEntryInstr;
260
261	/** IDT-vectoring information field. */
262	uint32_t uIdtVectoringInfo;
263	/** IDT-vectoring error code. */
264	uint32_t uIdtVectoringErrorCode;
265
266	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
267	uint32_t fVmcsFieldsRead;
268
269	/** Whether the guest FPU was active at the time of VM-exit. */
270	bool fWasGuestFPUStateActive;
271	/** Whether the guest debug state was active at the time of VM-exit. */
272	bool fWasGuestDebugStateActive;
273	/** Whether the hyper debug state was active at the time of VM-exit. */
274	bool fWasHyperDebugStateActive;
275	/** Whether TSC-offsetting should be setup before VM-entry. */
276	bool fUpdateTscOffsettingAndPreemptTimer;
277	/** Whether the VM-exit was caused by a page-fault during delivery of a
278	* contributory exception or a page-fault. */
279	bool fVectoringPF;
280	} VMXTRANSIENT;
281	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286	/** Pointer to VMX transient state. */
287	typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290	/**
291	* MSR-bitmap read permissions.
292	*/
293	typedef enum VMXMSREXITREAD
294	{
295	/** Reading this MSR causes a VM-exit. */
296	VMXMSREXIT_INTERCEPT_READ = 0xb,
297	/** Reading this MSR does not cause a VM-exit. */
298	VMXMSREXIT_PASSTHRU_READ
299	} VMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311
312	/**
313	* VMX VM-exit handler.
314	*
315	* @returns VBox status code.
316	* @param pVCpu Pointer to the VMCPU.
317	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
318	* out-of-sync. Make sure to update the required
319	* fields before using them.
320	* @param pVmxTransient Pointer to the VMX-transient structure.
321	*/
322	#ifndef HMVMX_USE_FUNCTION_TABLE
323	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
324	#else
325	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
326	/** Pointer to VM-exit handler. */
327	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
328	#endif
329
330
331	/*******************************************************************************
332	* Internal Functions *
333	*******************************************************************************/
334	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
335	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
336	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
337	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
338	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
339	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
340	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
341	#endif
342	#ifndef HMVMX_USE_FUNCTION_TABLE
343	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
344	# define HMVMX_EXIT_DECL static int
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
347	#endif
348
349	/** @name VM-exit handlers.
350	* @{
351	*/
352	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
353	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
354	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
355	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
356	static FNVMXEXITHANDLER hmR0VmxExitSipi;
357	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
358	static FNVMXEXITHANDLER hmR0VmxExitSmi;
359	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
360	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
361	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
362	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
363	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
364	static FNVMXEXITHANDLER hmR0VmxExitHlt;
365	static FNVMXEXITHANDLER hmR0VmxExitInvd;
366	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
367	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
368	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
369	static FNVMXEXITHANDLER hmR0VmxExitRsm;
370	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
371	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
372	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
373	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
374	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
375	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
376	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
377	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
378	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
379	static FNVMXEXITHANDLER hmR0VmxExitMwait;
380	static FNVMXEXITHANDLER hmR0VmxExitMtf;
381	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
382	static FNVMXEXITHANDLER hmR0VmxExitPause;
383	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
384	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
385	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
386	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
387	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
389	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
390	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
391	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
392	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
393	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
394	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
395	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
396	/** @} */
397
398	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
399	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
400	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
406
407	/*******************************************************************************
408	* Global Variables *
409	*******************************************************************************/
410	#ifdef HMVMX_USE_FUNCTION_TABLE
411
412	/**
413	* VMX_EXIT dispatch table.
414	*/
415	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
416	{
417	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
418	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
419	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
420	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
421	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
422	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
423	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
424	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
425	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
426	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
427	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
428	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
429	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
430	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
431	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
432	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
433	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
434	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
435	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
436	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
437	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
438	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
439	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
440	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
441	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
442	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
443	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
444	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
445	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
446	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
447	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
448	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
449	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
450	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
451	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
452	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
453	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
454	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
455	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
456	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
457	/* 40 UNDEFINED */ hmR0VmxExitPause,
458	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
459	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
460	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
461	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
462	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
463	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
464	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
465	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
466	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
467	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
468	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
469	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
470	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
471	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
472	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
473	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
474	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
475	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
476	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
477	};
478	#endif /* HMVMX_USE_FUNCTION_TABLE */
479
480	#ifdef VBOX_STRICT
481	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
482	{
483	/* 0 */ "(Not Used)",
484	/* 1 */ "VMCALL executed in VMX root operation.",
485	/* 2 */ "VMCLEAR with invalid physical address.",
486	/* 3 */ "VMCLEAR with VMXON pointer.",
487	/* 4 */ "VMLAUNCH with non-clear VMCS.",
488	/* 5 */ "VMRESUME with non-launched VMCS.",
489	/* 6 */ "VMRESUME after VMXOFF",
490	/* 7 */ "VM entry with invalid control fields.",
491	/* 8 */ "VM entry with invalid host state fields.",
492	/* 9 */ "VMPTRLD with invalid physical address.",
493	/* 10 */ "VMPTRLD with VMXON pointer.",
494	/* 11 */ "VMPTRLD with incorrect revision identifier.",
495	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
496	/* 13 */ "VMWRITE to read-only VMCS component.",
497	/* 14 */ "(Not Used)",
498	/* 15 */ "VMXON executed in VMX root operation.",
499	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
500	/* 17 */ "VM entry with non-launched executing VMCS.",
501	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
502	/* 19 */ "VMCALL with non-clear VMCS.",
503	/* 20 */ "VMCALL with invalid VM-exit control fields.",
504	/* 21 */ "(Not Used)",
505	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
506	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
507	/* 24 */ "VMCALL with invalid SMM-monitor features.",
508	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
509	/* 26 */ "VM entry with events blocked by MOV SS.",
510	/* 27 */ "(Not Used)",
511	/* 28 */ "Invalid operand to INVEPT/INVVPID."
512	};
513	#endif /* VBOX_STRICT */
514
515
516
517	/**
518	* Updates the VM's last error record. If there was a VMX instruction error,
519	* reads the error data from the VMCS and updates VCPU's last error record as
520	* well.
521	*
522	* @param pVM Pointer to the VM.
523	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
524	* VERR_VMX_UNABLE_TO_START_VM or
525	* VERR_VMX_INVALID_VMCS_FIELD).
526	* @param rc The error code.
527	*/
528	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
529	{
530	AssertPtr(pVM);
531	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
532	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
533	{
534	AssertPtrReturnVoid(pVCpu);
535	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
536	}
537	pVM->hm.s.lLastError = rc;
538	}
539
540
541	/**
542	* Reads the VM-entry interruption-information field from the VMCS into the VMX
543	* transient structure.
544	*
545	* @returns VBox status code.
546	* @param pVmxTransient Pointer to the VMX transient structure.
547	*
548	* @remarks No-long-jump zone!!!
549	*/
550	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
551	{
552	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
553	AssertRCReturn(rc, rc);
554	return VINF_SUCCESS;
555	}
556
557
558	/**
559	* Reads the VM-entry exception error code field from the VMCS into
560	* the VMX transient structure.
561	*
562	* @returns VBox status code.
563	* @param pVmxTransient Pointer to the VMX transient structure.
564	*
565	* @remarks No-long-jump zone!!!
566	*/
567	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
568	{
569	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
570	AssertRCReturn(rc, rc);
571	return VINF_SUCCESS;
572	}
573
574
575	/**
576	* Reads the VM-entry exception error code field from the VMCS into
577	* the VMX transient structure.
578	*
579	* @returns VBox status code.
580	* @param pVCpu Pointer to the VMCPU.
581	* @param pVmxTransient Pointer to the VMX transient structure.
582	*
583	* @remarks No-long-jump zone!!!
584	*/
585	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
586	{
587	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
588	AssertRCReturn(rc, rc);
589	return VINF_SUCCESS;
590	}
591
592
593	/**
594	* Reads the VM-exit interruption-information field from the VMCS into the VMX
595	* transient structure.
596	*
597	* @returns VBox status code.
598	* @param pVCpu Pointer to the VMCPU.
599	* @param pVmxTransient Pointer to the VMX transient structure.
600	*/
601	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
602	{
603	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
604	{
605	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
606	AssertRCReturn(rc, rc);
607	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
608	}
609	return VINF_SUCCESS;
610	}
611
612
613	/**
614	* Reads the VM-exit interruption error code from the VMCS into the VMX
615	* transient structure.
616	*
617	* @returns VBox status code.
618	* @param pVCpu Pointer to the VMCPU.
619	* @param pVmxTransient Pointer to the VMX transient structure.
620	*/
621	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
622	{
623	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
624	{
625	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
626	AssertRCReturn(rc, rc);
627	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
628	}
629	return VINF_SUCCESS;
630	}
631
632
633	/**
634	* Reads the VM-exit instruction length field from the VMCS into the VMX
635	* transient structure.
636	*
637	* @returns VBox status code.
638	* @param pVCpu Pointer to the VMCPU.
639	* @param pVmxTransient Pointer to the VMX transient structure.
640	*/
641	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
642	{
643	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
644	{
645	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
646	AssertRCReturn(rc, rc);
647	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
648	}
649	return VINF_SUCCESS;
650	}
651
652
653	/**
654	* Reads the VM-exit instruction-information field from the VMCS into
655	* the VMX transient structure.
656	*
657	* @returns VBox status code.
658	* @param pVCpu The cross context per CPU structure.
659	* @param pVmxTransient Pointer to the VMX transient structure.
660	*/
661	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
662	{
663	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664	{
665	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666	AssertRCReturn(rc, rc);
667	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668	}
669	return VINF_SUCCESS;
670	}
671
672
673	/**
674	* Reads the exit qualification from the VMCS into the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVCpu Pointer to the VMCPU.
678	* @param pVmxTransient Pointer to the VMX transient structure.
679	*/
680	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
681	{
682	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
683	{
684	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
685	AssertRCReturn(rc, rc);
686	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
687	}
688	return VINF_SUCCESS;
689	}
690
691
692	/**
693	* Reads the IDT-vectoring information field from the VMCS into the VMX
694	* transient structure.
695	*
696	* @returns VBox status code.
697	* @param pVmxTransient Pointer to the VMX transient structure.
698	*
699	* @remarks No-long-jump zone!!!
700	*/
701	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
702	{
703	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
704	{
705	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
706	AssertRCReturn(rc, rc);
707	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
708	}
709	return VINF_SUCCESS;
710	}
711
712
713	/**
714	* Reads the IDT-vectoring error code from the VMCS into the VMX
715	* transient structure.
716	*
717	* @returns VBox status code.
718	* @param pVmxTransient Pointer to the VMX transient structure.
719	*/
720	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
721	{
722	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
723	{
724	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
725	AssertRCReturn(rc, rc);
726	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
727	}
728	return VINF_SUCCESS;
729	}
730
731
732	/**
733	* Enters VMX root mode operation on the current CPU.
734	*
735	* @returns VBox status code.
736	* @param pVM Pointer to the VM (optional, can be NULL, after
737	* a resume).
738	* @param HCPhysCpuPage Physical address of the VMXON region.
739	* @param pvCpuPage Pointer to the VMXON region.
740	*/
741	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
742	{
743	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
744	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
745	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
746
747	if (pVM)
748	{
749	/* Write the VMCS revision dword to the VMXON region. */
750	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
751	}
752
753	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
754	RTCCUINTREG uEflags = ASMIntDisableFlags();
755
756	/* Enable the VMX bit in CR4 if necessary. */
757	RTCCUINTREG uCr4 = ASMGetCR4();
758	if (!(uCr4 & X86_CR4_VMXE))
759	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
760
761	/* Enter VMX root mode. */
762	int rc = VMXEnable(HCPhysCpuPage);
763	if (RT_FAILURE(rc))
764	ASMSetCR4(uCr4);
765
766	/* Restore interrupts. */
767	ASMSetFlags(uEflags);
768	return rc;
769	}
770
771
772	/**
773	* Exits VMX root mode operation on the current CPU.
774	*
775	* @returns VBox status code.
776	*/
777	static int hmR0VmxLeaveRootMode(void)
778	{
779	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
780
781	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
782	RTCCUINTREG uEflags = ASMIntDisableFlags();
783
784	/* If we're for some reason not in VMX root mode, then don't leave it. */
785	RTCCUINTREG uHostCR4 = ASMGetCR4();
786
787	int rc;
788	if (uHostCR4 & X86_CR4_VMXE)
789	{
790	/* Exit VMX root mode and clear the VMX bit in CR4. */
791	VMXDisable();
792	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
793	rc = VINF_SUCCESS;
794	}
795	else
796	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
797
798	/* Restore interrupts. */
799	ASMSetFlags(uEflags);
800	return rc;
801	}
802
803
804	/**
805	* Allocates and maps one physically contiguous page. The allocated page is
806	* zero'd out. (Used by various VT-x structures).
807	*
808	* @returns IPRT status code.
809	* @param pMemObj Pointer to the ring-0 memory object.
810	* @param ppVirt Where to store the virtual address of the
811	* allocation.
812	* @param pPhys Where to store the physical address of the
813	* allocation.
814	*/
815	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
816	{
817	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
818	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
819	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
820
821	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
822	if (RT_FAILURE(rc))
823	return rc;
824	ppVirt = RTR0MemObjAddress(pMemObj);
825	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
826	ASMMemZero32(*ppVirt, PAGE_SIZE);
827	return VINF_SUCCESS;
828	}
829
830
831	/**
832	* Frees and unmaps an allocated physical page.
833	*
834	* @param pMemObj Pointer to the ring-0 memory object.
835	* @param ppVirt Where to re-initialize the virtual address of
836	* allocation as 0.
837	* @param pHCPhys Where to re-initialize the physical address of the
838	* allocation as 0.
839	*/
840	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
841	{
842	AssertPtr(pMemObj);
843	AssertPtr(ppVirt);
844	AssertPtr(pHCPhys);
845	if (*pMemObj != NIL_RTR0MEMOBJ)
846	{
847	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
848	AssertRC(rc);
849	*pMemObj = NIL_RTR0MEMOBJ;
850	*ppVirt = 0;
851	*pHCPhys = 0;
852	}
853	}
854
855
856	/**
857	* Worker function to free VT-x related structures.
858	*
859	* @returns IPRT status code.
860	* @param pVM Pointer to the VM.
861	*/
862	static void hmR0VmxStructsFree(PVM pVM)
863	{
864	for (VMCPUID i = 0; i < pVM->cCpus; i++)
865	{
866	PVMCPU pVCpu = &pVM->aCpus[i];
867	AssertPtr(pVCpu);
868
869	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872	#endif
873
874	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
875	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
876
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
878	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
879	}
880
881	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
882	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
883	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
884	#endif
885	}
886
887
888	/**
889	* Worker function to allocate VT-x related VM structures.
890	*
891	* @returns IPRT status code.
892	* @param pVM Pointer to the VM.
893	*/
894	static int hmR0VmxStructsAlloc(PVM pVM)
895	{
896	/*
897	* Initialize members up-front so we can cleanup properly on allocation failure.
898	*/
899	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
900	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
901	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
902	pVM->hm.s.vmx.HCPhys##a_Name = 0;
903
904	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
905	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
908
909	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
910	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
911	#endif
912	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
913
914	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
915	for (VMCPUID i = 0; i < pVM->cCpus; i++)
916	{
917	PVMCPU pVCpu = &pVM->aCpus[i];
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
921	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
922	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
923	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
924	#endif
925	}
926	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
927	#undef VMXLOCAL_INIT_VM_MEMOBJ
928
929	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
930	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
931	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
932	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
933
934	/*
935	* Allocate all the VT-x structures.
936	*/
937	int rc = VINF_SUCCESS;
938	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
939	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
940	if (RT_FAILURE(rc))
941	goto cleanup;
942	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
943	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
944	#endif
945
946	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
947	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
948	{
949	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
950	&pVM->hm.s.vmx.HCPhysApicAccess);
951	if (RT_FAILURE(rc))
952	goto cleanup;
953	}
954
955	/*
956	* Initialize per-VCPU VT-x structures.
957	*/
958	for (VMCPUID i = 0; i < pVM->cCpus; i++)
959	{
960	PVMCPU pVCpu = &pVM->aCpus[i];
961	AssertPtr(pVCpu);
962
963	/* Allocate the VM control structure (VMCS). */
964	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
965	if (RT_FAILURE(rc))
966	goto cleanup;
967
968	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
969	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
970	{
971	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
972	&pVCpu->hm.s.vmx.HCPhysVirtApic);
973	if (RT_FAILURE(rc))
974	goto cleanup;
975	}
976
977	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
978	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
979	{
980	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
981	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
982	if (RT_FAILURE(rc))
983	goto cleanup;
984	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
985	}
986
987	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
988	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
989	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
990	if (RT_FAILURE(rc))
991	goto cleanup;
992
993	/* Allocate the VM-exit MSR-load page for the host MSRs. */
994	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
995	if (RT_FAILURE(rc))
996	goto cleanup;
997	#endif
998	}
999
1000	return VINF_SUCCESS;
1001
1002	cleanup:
1003	hmR0VmxStructsFree(pVM);
1004	return rc;
1005	}
1006
1007
1008	/**
1009	* Does global VT-x initialization (called during module initialization).
1010	*
1011	* @returns VBox status code.
1012	*/
1013	VMMR0DECL(int) VMXR0GlobalInit(void)
1014	{
1015	#ifdef HMVMX_USE_FUNCTION_TABLE
1016	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1017	# ifdef VBOX_STRICT
1018	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1019	Assert(g_apfnVMExitHandlers[i]);
1020	# endif
1021	#endif
1022	return VINF_SUCCESS;
1023	}
1024
1025
1026	/**
1027	* Does global VT-x termination (called during module termination).
1028	*/
1029	VMMR0DECL(void) VMXR0GlobalTerm()
1030	{
1031	/* Nothing to do currently. */
1032	}
1033
1034
1035	/**
1036	* Sets up and activates VT-x on the current CPU.
1037	*
1038	* @returns VBox status code.
1039	* @param pCpu Pointer to the global CPU info struct.
1040	* @param pVM Pointer to the VM (can be NULL after a host resume
1041	* operation).
1042	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1043	* fEnabledByHost is true).
1044	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1045	* @a fEnabledByHost is true).
1046	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1047	* enable VT-x on the host.
1048	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1049	*/
1050	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1051	void *pvMsrs)
1052	{
1053	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1054	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1055	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1056
1057	/* Enable VT-x if it's not already enabled by the host. */
1058	if (!fEnabledByHost)
1059	{
1060	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1061	if (RT_FAILURE(rc))
1062	return rc;
1063	}
1064
1065	/*
1066	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1067	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1068	*/
1069	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1070	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1071	{
1072	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1073	pCpu->fFlushAsidBeforeUse = false;
1074	}
1075	else
1076	pCpu->fFlushAsidBeforeUse = true;
1077
1078	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1079	++pCpu->cTlbFlushes;
1080
1081	return VINF_SUCCESS;
1082	}
1083
1084
1085	/**
1086	* Deactivates VT-x on the current CPU.
1087	*
1088	* @returns VBox status code.
1089	* @param pCpu Pointer to the global CPU info struct.
1090	* @param pvCpuPage Pointer to the VMXON region.
1091	* @param HCPhysCpuPage Physical address of the VMXON region.
1092	*
1093	* @remarks This function should never be called when SUPR0EnableVTx() or
1094	* similar was used to enable VT-x on the host.
1095	*/
1096	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1097	{
1098	NOREF(pCpu);
1099	NOREF(pvCpuPage);
1100	NOREF(HCPhysCpuPage);
1101
1102	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1103	return hmR0VmxLeaveRootMode();
1104	}
1105
1106
1107	/**
1108	* Sets the permission bits for the specified MSR in the MSR bitmap.
1109	*
1110	* @param pVCpu Pointer to the VMCPU.
1111	* @param uMSR The MSR value.
1112	* @param enmRead Whether reading this MSR causes a VM-exit.
1113	* @param enmWrite Whether writing this MSR causes a VM-exit.
1114	*/
1115	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1116	{
1117	int32_t iBit;
1118	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1119
1120	/*
1121	* Layout:
1122	* 0x000 - 0x3ff - Low MSR read bits
1123	* 0x400 - 0x7ff - High MSR read bits
1124	* 0x800 - 0xbff - Low MSR write bits
1125	* 0xc00 - 0xfff - High MSR write bits
1126	*/
1127	if (uMsr <= 0x00001FFF)
1128	iBit = uMsr;
1129	else if ( uMsr >= 0xC0000000
1130	&& uMsr <= 0xC0001FFF)
1131	{
1132	iBit = (uMsr - 0xC0000000);
1133	pbMsrBitmap += 0x400;
1134	}
1135	else
1136	{
1137	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1138	return;
1139	}
1140
1141	Assert(iBit <= 0x1fff);
1142	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1143	ASMBitSet(pbMsrBitmap, iBit);
1144	else
1145	ASMBitClear(pbMsrBitmap, iBit);
1146
1147	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1148	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1149	else
1150	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1151	}
1152
1153
1154	/**
1155	* Flushes the TLB using EPT.
1156	*
1157	* @returns VBox status code.
1158	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1159	* enmFlush).
1160	* @param enmFlush Type of flush.
1161	*
1162	* @remarks Caller is responsible for making sure this function is called only
1163	* when NestedPaging is supported and providing @a enmFlush that is
1164	* supported by the CPU.
1165	* @remarks Can be called with interrupts disabled.
1166	*/
1167	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1168	{
1169	uint64_t au64Descriptor[2];
1170	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1171	au64Descriptor[0] = 0;
1172	else
1173	{
1174	Assert(pVCpu);
1175	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1176	}
1177	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1178
1179	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1180	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1181	rc));
1182	if ( RT_SUCCESS(rc)
1183	&& pVCpu)
1184	{
1185	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1186	}
1187	}
1188
1189
1190	/**
1191	* Flushes the TLB using VPID.
1192	*
1193	* @returns VBox status code.
1194	* @param pVM Pointer to the VM.
1195	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1196	* enmFlush).
1197	* @param enmFlush Type of flush.
1198	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1199	* on @a enmFlush).
1200	*
1201	* @remarks Can be called with interrupts disabled.
1202	*/
1203	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1204	{
1205	AssertPtr(pVM);
1206	Assert(pVM->hm.s.vmx.fVpid);
1207
1208	uint64_t au64Descriptor[2];
1209	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1210	{
1211	au64Descriptor[0] = 0;
1212	au64Descriptor[1] = 0;
1213	}
1214	else
1215	{
1216	AssertPtr(pVCpu);
1217	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1218	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1219	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1220	au64Descriptor[1] = GCPtr;
1221	}
1222
1223	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1224	AssertMsg(rc == VINF_SUCCESS,
1225	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1226	if ( RT_SUCCESS(rc)
1227	&& pVCpu)
1228	{
1229	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1230	}
1231	}
1232
1233
1234	/**
1235	* Invalidates a guest page by guest virtual address. Only relevant for
1236	* EPT/VPID, otherwise there is nothing really to invalidate.
1237	*
1238	* @returns VBox status code.
1239	* @param pVM Pointer to the VM.
1240	* @param pVCpu Pointer to the VMCPU.
1241	* @param GCVirt Guest virtual address of the page to invalidate.
1242	*/
1243	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1244	{
1245	AssertPtr(pVM);
1246	AssertPtr(pVCpu);
1247	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1248
1249	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1250	if (!fFlushPending)
1251	{
1252	/*
1253	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1254	* See @bugref{6043} and @bugref{6177}.
1255	*
1256	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1257	* function maybe called in a loop with individual addresses.
1258	*/
1259	if (pVM->hm.s.vmx.fVpid)
1260	{
1261	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1262	{
1263	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1264	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1265	}
1266	else
1267	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1268	}
1269	else if (pVM->hm.s.fNestedPaging)
1270	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1271	}
1272
1273	return VINF_SUCCESS;
1274	}
1275
1276
1277	/**
1278	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1279	* otherwise there is nothing really to invalidate.
1280	*
1281	* @returns VBox status code.
1282	* @param pVM Pointer to the VM.
1283	* @param pVCpu Pointer to the VMCPU.
1284	* @param GCPhys Guest physical address of the page to invalidate.
1285	*/
1286	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1287	{
1288	LogFlowFunc(("%RGp\n", GCPhys));
1289
1290	/*
1291	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1292	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1293	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1294	*/
1295	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1296	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1297	return VINF_SUCCESS;
1298	}
1299
1300
1301	/**
1302	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1303	* case where neither EPT nor VPID is supported by the CPU.
1304	*
1305	* @param pVM Pointer to the VM.
1306	* @param pVCpu Pointer to the VMCPU.
1307	* @param pCpu Pointer to the global HM struct.
1308	*
1309	* @remarks Called with interrupts disabled.
1310	*/
1311	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1312	{
1313	AssertPtr(pVCpu);
1314	AssertPtr(pCpu);
1315	NOREF(pVM);
1316
1317	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1318
1319	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1320	#if 0
1321	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1322	pVCpu->hm.s.TlbShootdown.cPages = 0;
1323	#endif
1324
1325	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1326	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1327	pVCpu->hm.s.fForceTLBFlush = false;
1328	return;
1329	}
1330
1331
1332	/**
1333	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1334	*
1335	* @param pVM Pointer to the VM.
1336	* @param pVCpu Pointer to the VMCPU.
1337	* @param pCpu Pointer to the global HM CPU struct.
1338	* @remarks All references to "ASID" in this function pertains to "VPID" in
1339	* Intel's nomenclature. The reason is, to avoid confusion in compare
1340	* statements since the host-CPU copies are named "ASID".
1341	*
1342	* @remarks Called with interrupts disabled.
1343	*/
1344	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1345	{
1346	#ifdef VBOX_WITH_STATISTICS
1347	bool fTlbFlushed = false;
1348	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1349	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1350	if (!fTlbFlushed) \
1351	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1352	} while (0)
1353	#else
1354	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1355	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1356	#endif
1357
1358	AssertPtr(pVM);
1359	AssertPtr(pCpu);
1360	AssertPtr(pVCpu);
1361	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1362	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1363	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1364
1365
1366	/*
1367	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1368	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1369	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1370	*/
1371	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1372	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1373	{
1374	++pCpu->uCurrentAsid;
1375	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1376	{
1377	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1378	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1379	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1380	}
1381
1382	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1383	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1384	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1385
1386	/*
1387	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1388	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1389	*/
1390	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1391	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1392	HMVMX_SET_TAGGED_TLB_FLUSHED();
1393	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1394	}
1395
1396	/* Check for explicit TLB shootdowns. */
1397	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1398	{
1399	/*
1400	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1401	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1402	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1403	* but not guest-physical mappings.
1404	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1405	*/
1406	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1407	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1408	HMVMX_SET_TAGGED_TLB_FLUSHED();
1409	}
1410
1411	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1412	* where it is commented out. Support individual entry flushing
1413	* someday. */
1414	#if 0
1415	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1416	{
1417	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1418
1419	/*
1420	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1421	* as supported by the CPU.
1422	*/
1423	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1424	{
1425	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1426	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1427	}
1428	else
1429	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1430
1431	HMVMX_SET_TAGGED_TLB_FLUSHED();
1432	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1433	pVCpu->hm.s.TlbShootdown.cPages = 0;
1434	}
1435	#endif
1436
1437	pVCpu->hm.s.fForceTLBFlush = false;
1438
1439	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1440
1441	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1442	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1443	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1444	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1445	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1446	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1447	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1448	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1449
1450	/* Update VMCS with the VPID. */
1451	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1452	AssertRC(rc);
1453
1454	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1455	}
1456
1457
1458	/**
1459	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1460	*
1461	* @returns VBox status code.
1462	* @param pVM Pointer to the VM.
1463	* @param pVCpu Pointer to the VMCPU.
1464	* @param pCpu Pointer to the global HM CPU struct.
1465	*
1466	* @remarks Called with interrupts disabled.
1467	*/
1468	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1469	{
1470	AssertPtr(pVM);
1471	AssertPtr(pVCpu);
1472	AssertPtr(pCpu);
1473	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1474	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1475
1476	/*
1477	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1478	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1479	*/
1480	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1481	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1482	{
1483	pVCpu->hm.s.fForceTLBFlush = true;
1484	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1485	}
1486
1487	/* Check for explicit TLB shootdown flushes. */
1488	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1489	{
1490	pVCpu->hm.s.fForceTLBFlush = true;
1491	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1492	}
1493
1494	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1495	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1496
1497	if (pVCpu->hm.s.fForceTLBFlush)
1498	{
1499	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1500	pVCpu->hm.s.fForceTLBFlush = false;
1501	}
1502	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1503	* where it is commented out. Support individual entry flushing
1504	* someday. */
1505	#if 0
1506	else
1507	{
1508	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1509	{
1510	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1511	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1512	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1513	}
1514	else
1515	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1516
1517	pVCpu->hm.s.TlbShootdown.cPages = 0;
1518	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1519	}
1520	#endif
1521	}
1522
1523
1524	/**
1525	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1526	*
1527	* @returns VBox status code.
1528	* @param pVM Pointer to the VM.
1529	* @param pVCpu Pointer to the VMCPU.
1530	* @param pCpu Pointer to the global HM CPU struct.
1531	*
1532	* @remarks Called with interrupts disabled.
1533	*/
1534	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1535	{
1536	AssertPtr(pVM);
1537	AssertPtr(pVCpu);
1538	AssertPtr(pCpu);
1539	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1540	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1541
1542	/*
1543	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1544	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1545	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1546	*/
1547	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1548	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1549	{
1550	pVCpu->hm.s.fForceTLBFlush = true;
1551	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1552	}
1553
1554	/* Check for explicit TLB shootdown flushes. */
1555	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1556	{
1557	/*
1558	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1559	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1560	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1561	*/
1562	pVCpu->hm.s.fForceTLBFlush = true;
1563	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1564	}
1565
1566	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1567	if (pVCpu->hm.s.fForceTLBFlush)
1568	{
1569	++pCpu->uCurrentAsid;
1570	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1571	{
1572	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1573	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1574	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1575	}
1576
1577	pVCpu->hm.s.fForceTLBFlush = false;
1578	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1579	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1580	if (pCpu->fFlushAsidBeforeUse)
1581	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1582	}
1583	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1584	* where it is commented out. Support individual entry flushing
1585	* someday. */
1586	#if 0
1587	else
1588	{
1589	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1590	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1591	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1592	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1593
1594	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1595	{
1596	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1597	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1598	{
1599	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1600	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1601	}
1602	else
1603	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1604
1605	pVCpu->hm.s.TlbShootdown.cPages = 0;
1606	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1607	}
1608	else
1609	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1610	}
1611	#endif
1612
1613	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1614	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1615	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1616	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1617	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1618	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1619
1620	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1621	AssertRC(rc);
1622	}
1623
1624
1625	/**
1626	* Flushes the guest TLB entry based on CPU capabilities.
1627	*
1628	* @param pVCpu Pointer to the VMCPU.
1629	* @param pCpu Pointer to the global HM CPU struct.
1630	*/
1631	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1632	{
1633	PVM pVM = pVCpu->CTX_SUFF(pVM);
1634	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1635	{
1636	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1637	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1638	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1639	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1640	default:
1641	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1642	break;
1643	}
1644	}
1645
1646
1647	/**
1648	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1649	* TLB entries from the host TLB before VM-entry.
1650	*
1651	* @returns VBox status code.
1652	* @param pVM Pointer to the VM.
1653	*/
1654	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1655	{
1656	/*
1657	* Determine optimal flush type for Nested Paging.
1658	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1659	* guest execution (see hmR3InitFinalizeR0()).
1660	*/
1661	if (pVM->hm.s.fNestedPaging)
1662	{
1663	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1664	{
1665	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1666	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1667	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1668	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1669	else
1670	{
1671	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1672	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1673	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1674	}
1675
1676	/* Make sure the write-back cacheable memory type for EPT is supported. */
1677	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1678	{
1679	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1680	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1681	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1682	}
1683	}
1684	else
1685	{
1686	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1687	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1688	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1689	}
1690	}
1691
1692	/*
1693	* Determine optimal flush type for VPID.
1694	*/
1695	if (pVM->hm.s.vmx.fVpid)
1696	{
1697	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1698	{
1699	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1700	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1701	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1702	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1703	else
1704	{
1705	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1706	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1707	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1708	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1709	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1710	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1711	pVM->hm.s.vmx.fVpid = false;
1712	}
1713	}
1714	else
1715	{
1716	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1717	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1718	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1719	pVM->hm.s.vmx.fVpid = false;
1720	}
1721	}
1722
1723	/*
1724	* Setup the handler for flushing tagged-TLBs.
1725	*/
1726	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1727	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1728	else if (pVM->hm.s.fNestedPaging)
1729	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1730	else if (pVM->hm.s.vmx.fVpid)
1731	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1732	else
1733	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1734	return VINF_SUCCESS;
1735	}
1736
1737
1738	/**
1739	* Sets up pin-based VM-execution controls in the VMCS.
1740	*
1741	* @returns VBox status code.
1742	* @param pVM Pointer to the VM.
1743	* @param pVCpu Pointer to the VMCPU.
1744	*/
1745	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1746	{
1747	AssertPtr(pVM);
1748	AssertPtr(pVCpu);
1749
1750	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1751	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1752
1753	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1754	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1755	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1756
1757	/* Enable the VMX preemption timer. */
1758	if (pVM->hm.s.vmx.fUsePreemptTimer)
1759	{
1760	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1761	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1762	}
1763
1764	if ((val & zap) != val)
1765	{
1766	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1767	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1768	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1769	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1770	}
1771
1772	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1773	AssertRCReturn(rc, rc);
1774
1775	/* Update VCPU with the currently set pin-based VM-execution controls. */
1776	pVCpu->hm.s.vmx.u32PinCtls = val;
1777	return rc;
1778	}
1779
1780
1781	/**
1782	* Sets up processor-based VM-execution controls in the VMCS.
1783	*
1784	* @returns VBox status code.
1785	* @param pVM Pointer to the VM.
1786	* @param pVMCPU Pointer to the VMCPU.
1787	*/
1788	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1789	{
1790	AssertPtr(pVM);
1791	AssertPtr(pVCpu);
1792
1793	int rc = VERR_INTERNAL_ERROR_5;
1794	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1795	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1796
1797	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1798	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1799	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1800	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1801	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1802	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1803	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1804
1805	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1806	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1807	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1808	{
1809	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1810	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1811	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1812	}
1813
1814	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1815	if (!pVM->hm.s.fNestedPaging)
1816	{
1817	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1818	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1819	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1820	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1821	}
1822
1823	/* Use TPR shadowing if supported by the CPU. */
1824	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1825	{
1826	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1827	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1828	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1829	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1830	AssertRCReturn(rc, rc);
1831
1832	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1833	/* CR8 writes causes a VM-exit based on TPR threshold. */
1834	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1835	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1836	}
1837	else
1838	{
1839	/*
1840	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
1841	* Set this control only for 64-bit guests.
1842	*/
1843	if (pVM->hm.s.fAllow64BitGuests)
1844	{
1845	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1846	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1847	}
1848	}
1849
1850	/* Use MSR-bitmaps if supported by the CPU. */
1851	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1852	{
1853	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1854
1855	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1856	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1857	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1858	AssertRCReturn(rc, rc);
1859
1860	/*
1861	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1862	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1863	*/
1864	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1865	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1866	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1867	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1868	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1869	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1870	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1871	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1872	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1873	}
1874
1875	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1876	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1877	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1878
1879	if ((val & zap) != val)
1880	{
1881	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1882	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1883	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1884	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1885	}
1886
1887	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1888	AssertRCReturn(rc, rc);
1889
1890	/* Update VCPU with the currently set processor-based VM-execution controls. */
1891	pVCpu->hm.s.vmx.u32ProcCtls = val;
1892
1893	/*
1894	* Secondary processor-based VM-execution controls.
1895	*/
1896	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1897	{
1898	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1899	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1900
1901	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1902	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1903
1904	if (pVM->hm.s.fNestedPaging)
1905	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1906	else
1907	{
1908	/*
1909	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1910	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1911	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1912	*/
1913	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1914	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1915	}
1916
1917	if (pVM->hm.s.vmx.fVpid)
1918	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1919
1920	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1921	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1922
1923	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1924	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1925	* done dynamically. */
1926	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1927	{
1928	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1929	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1930	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1931	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1932	AssertRCReturn(rc, rc);
1933	}
1934
1935	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1936	{
1937	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1938	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1939	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1940	}
1941
1942	if ((val & zap) != val)
1943	{
1944	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1945	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1946	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1947	}
1948
1949	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1950	AssertRCReturn(rc, rc);
1951
1952	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1953	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1954	}
1955	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1956	{
1957	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1958	"available\n"));
1959	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1960	}
1961
1962	return VINF_SUCCESS;
1963	}
1964
1965
1966	/**
1967	* Sets up miscellaneous (everything other than Pin & Processor-based
1968	* VM-execution) control fields in the VMCS.
1969	*
1970	* @returns VBox status code.
1971	* @param pVM Pointer to the VM.
1972	* @param pVCpu Pointer to the VMCPU.
1973	*/
1974	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1975	{
1976	AssertPtr(pVM);
1977	AssertPtr(pVCpu);
1978
1979	int rc = VERR_GENERAL_FAILURE;
1980
1981	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1982	#if 0
1983	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
1984	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1985	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1986
1987	/*
1988	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1989	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1990	* We thus use the exception bitmap to control it rather than use both.
1991	*/
1992	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1993	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1994
1995	/** @todo Explore possibility of using IO-bitmaps. */
1996	/* All IO & IOIO instructions cause VM-exits. */
1997	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1998	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1999
2000	/* Initialize the MSR-bitmap area. */
2001	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2002	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2003	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2004	#endif
2005
2006	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2007	/* Setup MSR autoloading/storing. */
2008	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2009	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2010	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2011	AssertRCReturn(rc, rc);
2012	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2013	AssertRCReturn(rc, rc);
2014
2015	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2016	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2017	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2018	AssertRCReturn(rc, rc);
2019	#endif
2020
2021	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2022	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2023	AssertRCReturn(rc, rc);
2024
2025	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2026	#if 0
2027	/* Setup debug controls */
2028	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2029	AssertRCReturn(rc, rc);
2030	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2031	AssertRCReturn(rc, rc);
2032	#endif
2033
2034	return rc;
2035	}
2036
2037
2038	/**
2039	* Sets up the initial exception bitmap in the VMCS based on static conditions
2040	* (i.e. conditions that cannot ever change after starting the VM).
2041	*
2042	* @returns VBox status code.
2043	* @param pVM Pointer to the VM.
2044	* @param pVCpu Pointer to the VMCPU.
2045	*/
2046	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2047	{
2048	AssertPtr(pVM);
2049	AssertPtr(pVCpu);
2050
2051	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2052
2053	uint32_t u32XcptBitmap = 0;
2054
2055	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2056	if (!pVM->hm.s.fNestedPaging)
2057	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2058
2059	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2060	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2061	AssertRCReturn(rc, rc);
2062	return rc;
2063	}
2064
2065
2066	/**
2067	* Sets up the initial guest-state mask. The guest-state mask is consulted
2068	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2069	* for the nested virtualization case (as it would cause a VM-exit).
2070	*
2071	* @param pVCpu Pointer to the VMCPU.
2072	*/
2073	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2074	{
2075	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2076	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2077	return VINF_SUCCESS;
2078	}
2079
2080
2081	/**
2082	* Does per-VM VT-x initialization.
2083	*
2084	* @returns VBox status code.
2085	* @param pVM Pointer to the VM.
2086	*/
2087	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2088	{
2089	LogFlowFunc(("pVM=%p\n", pVM));
2090
2091	int rc = hmR0VmxStructsAlloc(pVM);
2092	if (RT_FAILURE(rc))
2093	{
2094	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2095	return rc;
2096	}
2097
2098	return VINF_SUCCESS;
2099	}
2100
2101
2102	/**
2103	* Does per-VM VT-x termination.
2104	*
2105	* @returns VBox status code.
2106	* @param pVM Pointer to the VM.
2107	*/
2108	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2109	{
2110	LogFlowFunc(("pVM=%p\n", pVM));
2111
2112	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2113	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2114	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2115	#endif
2116	hmR0VmxStructsFree(pVM);
2117	return VINF_SUCCESS;
2118	}
2119
2120
2121	/**
2122	* Sets up the VM for execution under VT-x.
2123	* This function is only called once per-VM during initialization.
2124	*
2125	* @returns VBox status code.
2126	* @param pVM Pointer to the VM.
2127	*/
2128	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2129	{
2130	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2131	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2132
2133	LogFlowFunc(("pVM=%p\n", pVM));
2134
2135	/*
2136	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2137	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2138	*/
2139	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2140	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2141	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2142	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2143	{
2144	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2145	return VERR_INTERNAL_ERROR;
2146	}
2147
2148	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2149	/*
2150	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2151	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2152	*/
2153	if ( pVM->hm.s.fAllow64BitGuests
2154	&& !HMVMX_IS_64BIT_HOST_MODE())
2155	{
2156	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2157	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2158	}
2159	#endif
2160
2161	/* Initialize these always, see hmR3InitFinalizeR0().*/
2162	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2163	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2164
2165	/* Setup the tagged-TLB flush handlers. */
2166	int rc = hmR0VmxSetupTaggedTlb(pVM);
2167	if (RT_FAILURE(rc))
2168	{
2169	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2170	return rc;
2171	}
2172
2173	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2174	{
2175	PVMCPU pVCpu = &pVM->aCpus[i];
2176	AssertPtr(pVCpu);
2177	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2178
2179	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2180	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2181
2182	/* Set revision dword at the beginning of the VMCS structure. */
2183	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2184
2185	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2186	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2187	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2188	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2189
2190	/* Load this VMCS as the current VMCS. */
2191	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2192	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2193	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2194
2195	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2196	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2197	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2198
2199	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2200	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2201	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2202
2203	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2204	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2205	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2206
2207	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2208	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2209	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2210
2211	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2212	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2213	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2214
2215	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2216	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2217	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2218	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2219	#endif
2220
2221	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2222	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2223	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2224	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2225
2226	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2227
2228	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2229	}
2230
2231	return VINF_SUCCESS;
2232	}
2233
2234
2235	/**
2236	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2237	* the VMCS.
2238	*
2239	* @returns VBox status code.
2240	* @param pVM Pointer to the VM.
2241	* @param pVCpu Pointer to the VMCPU.
2242	*/
2243	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2244	{
2245	RTCCUINTREG uReg = ASMGetCR0();
2246	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2247	AssertRCReturn(rc, rc);
2248
2249	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2250	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2251	if (HMVMX_IS_64BIT_HOST_MODE())
2252	{
2253	uint64_t uRegCR3 = HMR0Get64bitCR3();
2254	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2255	}
2256	else
2257	#endif
2258	{
2259	uReg = ASMGetCR3();
2260	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2261	}
2262	AssertRCReturn(rc, rc);
2263
2264	uReg = ASMGetCR4();
2265	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2266	AssertRCReturn(rc, rc);
2267	return rc;
2268	}
2269
2270
2271	#if HC_ARCH_BITS == 64
2272	/**
2273	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2274	* requirements. See hmR0VmxSaveHostSegmentRegs().
2275	*/
2276	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2277	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2278	{ \
2279	bool fValidSelector = true; \
2280	if ((selValue) & X86_SEL_LDT) \
2281	{ \
2282	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2283	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2284	} \
2285	if (fValidSelector) \
2286	{ \
2287	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2288	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2289	} \
2290	(selValue) = 0; \
2291	}
2292	#endif
2293
2294
2295	/**
2296	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2297	* the host-state area in the VMCS.
2298	*
2299	* @returns VBox status code.
2300	* @param pVM Pointer to the VM.
2301	* @param pVCpu Pointer to the VMCPU.
2302	*/
2303	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2304	{
2305	int rc = VERR_INTERNAL_ERROR_5;
2306
2307	/*
2308	* Host DS, ES, FS and GS segment registers.
2309	*/
2310	#if HC_ARCH_BITS == 64
2311	RTSEL uSelDS = ASMGetDS();
2312	RTSEL uSelES = ASMGetES();
2313	RTSEL uSelFS = ASMGetFS();
2314	RTSEL uSelGS = ASMGetGS();
2315	#else
2316	RTSEL uSelDS = 0;
2317	RTSEL uSelES = 0;
2318	RTSEL uSelFS = 0;
2319	RTSEL uSelGS = 0;
2320	#endif
2321
2322	/* Recalculate which host-state bits need to be manually restored. */
2323	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2324
2325	/*
2326	* Host CS and SS segment registers.
2327	*/
2328	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2329	RTSEL uSelCS;
2330	RTSEL uSelSS;
2331	if (HMVMX_IS_64BIT_HOST_MODE())
2332	{
2333	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2334	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2335	}
2336	else
2337	{
2338	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2339	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2340	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2341	}
2342	#else
2343	RTSEL uSelCS = ASMGetCS();
2344	RTSEL uSelSS = ASMGetSS();
2345	#endif
2346
2347	/*
2348	* Host TR segment register.
2349	*/
2350	RTSEL uSelTR = ASMGetTR();
2351
2352	#if HC_ARCH_BITS == 64
2353	/*
2354	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2355	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2356	*/
2357	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2358	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2359	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2360	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2361	# undef VMXLOCAL_ADJUST_HOST_SEG
2362	#endif
2363
2364	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2365	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2366	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2367	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2368	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2369	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2370	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2371	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2372	Assert(uSelCS);
2373	Assert(uSelTR);
2374
2375	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2376	#if 0
2377	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2378	Assert(uSelSS != 0);
2379	#endif
2380
2381	/* Write these host selector fields into the host-state area in the VMCS. */
2382	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2383	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2384	#if HC_ARCH_BITS == 64
2385	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2386	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2387	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2388	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2389	#endif
2390	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2391
2392	/*
2393	* Host GDTR and IDTR.
2394	*/
2395	RTGDTR Gdtr;
2396	RT_ZERO(Gdtr);
2397	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2398	if (HMVMX_IS_64BIT_HOST_MODE())
2399	{
2400	X86XDTR64 Gdtr64;
2401	X86XDTR64 Idtr64;
2402	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2403	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2404	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2405
2406	Gdtr.cbGdt = Gdtr64.cb;
2407	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2408	}
2409	else
2410	#endif
2411	{
2412	RTIDTR Idtr;
2413	ASMGetGDTR(&Gdtr);
2414	ASMGetIDTR(&Idtr);
2415	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2416	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2417
2418	#if HC_ARCH_BITS == 64
2419	/*
2420	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2421	* maximum limit (0xffff) on every VM-exit.
2422	*/
2423	if (Gdtr.cbGdt != 0xffff)
2424	{
2425	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2426	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2427	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2428	}
2429
2430	/*
2431	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2432	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2433	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2434	*/
2435	if (Idtr.cbIdt < 0x0fff)
2436	{
2437	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2438	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2439	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2440	}
2441	#endif
2442	}
2443
2444	/*
2445	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2446	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2447	*/
2448	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2449	{
2450	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2451	return VERR_VMX_INVALID_HOST_STATE;
2452	}
2453
2454	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2455	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2456	if (HMVMX_IS_64BIT_HOST_MODE())
2457	{
2458	/* We need the 64-bit TR base for hybrid darwin. */
2459	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2460	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2461	}
2462	else
2463	#endif
2464	{
2465	uintptr_t uTRBase;
2466	#if HC_ARCH_BITS == 64
2467	uTRBase = X86DESC64_BASE(pDesc);
2468
2469	/*
2470	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2471	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2472	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2473	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2474	*
2475	* [1] See Intel spec. 3.5 "System Descriptor Types".
2476	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2477	*/
2478	Assert(pDesc->System.u4Type == 11);
2479	if ( pDesc->System.u16LimitLow != 0x67
2480	\|\| pDesc->System.u4LimitHigh)
2481	{
2482	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2483	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2484
2485	/* Store the GDTR here as we need it while restoring TR. */
2486	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2487	}
2488	#else
2489	uTRBase = X86DESC_BASE(pDesc);
2490	#endif
2491	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2492	}
2493	AssertRCReturn(rc, rc);
2494
2495	/*
2496	* Host FS base and GS base.
2497	*/
2498	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2499	if (HMVMX_IS_64BIT_HOST_MODE())
2500	{
2501	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2502	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2503	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2504	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2505
2506	# if HC_ARCH_BITS == 64
2507	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2508	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2509	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2510	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2511	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2512	# endif
2513	}
2514	#endif
2515	return rc;
2516	}
2517
2518
2519	/**
2520	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2521	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2522	* the host after every successful VM exit.
2523	*
2524	* @returns VBox status code.
2525	* @param pVM Pointer to the VM.
2526	* @param pVCpu Pointer to the VMCPU.
2527	*/
2528	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2529	{
2530	AssertPtr(pVCpu);
2531	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2532
2533	int rc = VINF_SUCCESS;
2534	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2535	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2536	uint32_t cHostMsrs = 0;
2537	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2538
2539	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2540	{
2541	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2542
2543	# if HC_ARCH_BITS == 64
2544	/* Paranoia. 64-bit code requires these bits to be set always. */
2545	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2546
2547	/*
2548	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2549	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2550	* some reason (e.g. allow transparent reads) we would activate the code below.
2551	*/
2552	# if 0
2553	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2554	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2555	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2556	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2557	if (CPUMIsGuestInLongMode(pVCpu))
2558	{
2559	uint64_t u64GuestEfer;
2560	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2561	AssertRC(rc);
2562
2563	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2564	{
2565	pHostMsr->u32Msr = MSR_K6_EFER;
2566	pHostMsr->u32Reserved = 0;
2567	pHostMsr->u64Value = u64HostEfer;
2568	pHostMsr++; cHostMsrs++;
2569	}
2570	}
2571	# endif
2572	# else /* HC_ARCH_BITS != 64 */
2573	pHostMsr->u32Msr = MSR_K6_EFER;
2574	pHostMsr->u32Reserved = 0;
2575	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2576	if (CPUMIsGuestInLongMode(pVCpu))
2577	{
2578	/* Must match the EFER value in our 64 bits switcher. */
2579	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2580	}
2581	else
2582	# endif
2583	pHostMsr->u64Value = u64HostEfer;
2584	pHostMsr++; cHostMsrs++;
2585	# endif /* HC_ARCH_BITS == 64 */
2586	}
2587
2588	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2589	if (HMVMX_IS_64BIT_HOST_MODE())
2590	{
2591	pHostMsr->u32Msr = MSR_K6_STAR;
2592	pHostMsr->u32Reserved = 0;
2593	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2594	pHostMsr++; cHostMsrs++;
2595	pHostMsr->u32Msr = MSR_K8_LSTAR;
2596	pHostMsr->u32Reserved = 0;
2597	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2598	pHostMsr++; cHostMsrs++;
2599	pHostMsr->u32Msr = MSR_K8_SF_MASK;
2600	pHostMsr->u32Reserved = 0;
2601	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2602	pHostMsr++; cHostMsrs++;
2603	pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2604	pHostMsr->u32Reserved = 0;
2605	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2606	pHostMsr++; cHostMsrs++;
2607	}
2608	# endif
2609
2610	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2611	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2612	{
2613	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2614	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2615	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2616	}
2617
2618	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2619	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2620
2621	/*
2622	* Host Sysenter MSRs.
2623	*/
2624	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2625	AssertRCReturn(rc, rc);
2626	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2627	if (HMVMX_IS_64BIT_HOST_MODE())
2628	{
2629	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2630	AssertRCReturn(rc, rc);
2631	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2632	}
2633	else
2634	{
2635	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2636	AssertRCReturn(rc, rc);
2637	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2638	}
2639	#elif HC_ARCH_BITS == 32
2640	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2641	AssertRCReturn(rc, rc);
2642	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2643	#else
2644	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2645	AssertRCReturn(rc, rc);
2646	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2647	#endif
2648	AssertRCReturn(rc, rc);
2649
2650	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2651	* hmR0VmxSetupExitCtls() !! */
2652	return rc;
2653	}
2654
2655
2656	/**
2657	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2658	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2659	* controls".
2660	*
2661	* @returns VBox status code.
2662	* @param pVCpu Pointer to the VMCPU.
2663	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2664	* out-of-sync. Make sure to update the required fields
2665	* before using them.
2666	*
2667	* @remarks No-long-jump zone!!!
2668	*/
2669	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2670	{
2671	int rc = VINF_SUCCESS;
2672	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
2673	{
2674	PVM pVM = pVCpu->CTX_SUFF(pVM);
2675	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2676	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2677
2678	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2679	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2680
2681	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2682	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2683	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2684	else
2685	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2686
2687	/*
2688	* The following should -not- be set (since we're not in SMM mode):
2689	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2690	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2691	*/
2692
2693	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2694	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2695	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2696
2697	if ((val & zap) != val)
2698	{
2699	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2700	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2701	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2702	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2703	}
2704
2705	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2706	AssertRCReturn(rc, rc);
2707
2708	/* Update VCPU with the currently set VM-exit controls. */
2709	pVCpu->hm.s.vmx.u32EntryCtls = val;
2710	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
2711	}
2712	return rc;
2713	}
2714
2715
2716	/**
2717	* Sets up the VM-exit controls in the VMCS.
2718	*
2719	* @returns VBox status code.
2720	* @param pVM Pointer to the VM.
2721	* @param pVCpu Pointer to the VMCPU.
2722	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2723	* out-of-sync. Make sure to update the required fields
2724	* before using them.
2725	*
2726	* @remarks requires EFER.
2727	*/
2728	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2729	{
2730	int rc = VINF_SUCCESS;
2731	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
2732	{
2733	PVM pVM = pVCpu->CTX_SUFF(pVM);
2734	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2735	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2736
2737	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2738	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2739
2740	/*
2741	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2742	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2743	*/
2744	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2745	if (HMVMX_IS_64BIT_HOST_MODE())
2746	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2747	else
2748	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2749	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2750	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2751	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2752	else
2753	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2754	#endif
2755
2756	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2757	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2758
2759	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2760	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2761	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2762	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2763	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2764
2765	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2766	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2767
2768	if ((val & zap) != val)
2769	{
2770	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2771	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2772	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2773	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2774	}
2775
2776	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2777	AssertRCReturn(rc, rc);
2778
2779	/* Update VCPU with the currently set VM-exit controls. */
2780	pVCpu->hm.s.vmx.u32ExitCtls = val;
2781	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
2782	}
2783	return rc;
2784	}
2785
2786
2787	/**
2788	* Loads the guest APIC and related state.
2789	*
2790	* @returns VBox status code.
2791	* @param pVM Pointer to the VM.
2792	* @param pVCpu Pointer to the VMCPU.
2793	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2794	* out-of-sync. Make sure to update the required fields
2795	* before using them.
2796	*/
2797	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2798	{
2799	int rc = VINF_SUCCESS;
2800	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
2801	{
2802	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2803	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2804	{
2805	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2806
2807	bool fPendingIntr = false;
2808	uint8_t u8Tpr = 0;
2809	uint8_t u8PendingIntr = 0;
2810	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2811	AssertRCReturn(rc, rc);
2812
2813	/*
2814	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2815	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2816	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2817	* the interrupt when we VM-exit for other reasons.
2818	*/
2819	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2820	uint32_t u32TprThreshold = 0;
2821	if (fPendingIntr)
2822	{
2823	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2824	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2825	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2826	if (u8PendingPriority <= u8TprPriority)
2827	u32TprThreshold = u8PendingPriority;
2828	else
2829	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2830	}
2831	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2832
2833	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2834	AssertRCReturn(rc, rc);
2835	}
2836
2837	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
2838	}
2839	return rc;
2840	}
2841
2842
2843	/**
2844	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2845	*
2846	* @returns Guest's interruptibility-state.
2847	* @param pVCpu Pointer to the VMCPU.
2848	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2849	* out-of-sync. Make sure to update the required fields
2850	* before using them.
2851	*
2852	* @remarks No-long-jump zone!!!
2853	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2854	*/
2855	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2856	{
2857	/*
2858	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2859	* inhibit interrupts or clear any existing interrupt-inhibition.
2860	*/
2861	uint32_t uIntrState = 0;
2862	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2863	{
2864	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2865	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2866	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2867	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2868	{
2869	/*
2870	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2871	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2872	*/
2873	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2874	}
2875	else if (pMixedCtx->eflags.Bits.u1IF)
2876	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2877	else
2878	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2879	}
2880	return uIntrState;
2881	}
2882
2883
2884	/**
2885	* Loads the guest's interruptibility-state into the guest-state area in the
2886	* VMCS.
2887	*
2888	* @returns VBox status code.
2889	* @param pVCpu Pointer to the VMCPU.
2890	* @param uIntrState The interruptibility-state to set.
2891	*/
2892	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2893	{
2894	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2895	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2896	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2897	AssertRCReturn(rc, rc);
2898	return rc;
2899	}
2900
2901
2902	/**
2903	* Loads the guest's RIP into the guest-state area in the VMCS.
2904	*
2905	* @returns VBox status code.
2906	* @param pVCpu Pointer to the VMCPU.
2907	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2908	* out-of-sync. Make sure to update the required fields
2909	* before using them.
2910	*
2911	* @remarks No-long-jump zone!!!
2912	*/
2913	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2914	{
2915	int rc = VINF_SUCCESS;
2916	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
2917	{
2918	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2919	AssertRCReturn(rc, rc);
2920
2921	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
2922	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
2923	}
2924	return rc;
2925	}
2926
2927
2928	/**
2929	* Loads the guest's RSP into the guest-state area in the VMCS.
2930	*
2931	* @returns VBox status code.
2932	* @param pVCpu Pointer to the VMCPU.
2933	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2934	* out-of-sync. Make sure to update the required fields
2935	* before using them.
2936	*
2937	* @remarks No-long-jump zone!!!
2938	*/
2939	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2940	{
2941	int rc = VINF_SUCCESS;
2942	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
2943	{
2944	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2945	AssertRCReturn(rc, rc);
2946
2947	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
2948	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2949	}
2950	return rc;
2951	}
2952
2953
2954	/**
2955	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2956	*
2957	* @returns VBox status code.
2958	* @param pVCpu Pointer to the VMCPU.
2959	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2960	* out-of-sync. Make sure to update the required fields
2961	* before using them.
2962	*
2963	* @remarks No-long-jump zone!!!
2964	*/
2965	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2966	{
2967	int rc = VINF_SUCCESS;
2968	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
2969	{
2970	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2971	Let us assert it as such and use 32-bit VMWRITE. */
2972	Assert(!(pMixedCtx->rflags.u64 >> 32));
2973	X86EFLAGS Eflags = pMixedCtx->eflags;
2974	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2975	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2976
2977	/*
2978	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2979	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2980	*/
2981	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2982	{
2983	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2984	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2985	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2986	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2987	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2988	}
2989
2990	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2991	AssertRCReturn(rc, rc);
2992
2993	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
2994	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2995	}
2996	return rc;
2997	}
2998
2999
3000	/**
3001	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3002	*
3003	* @returns VBox status code.
3004	* @param pVCpu Pointer to the VMCPU.
3005	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3006	* out-of-sync. Make sure to update the required fields
3007	* before using them.
3008	*
3009	* @remarks No-long-jump zone!!!
3010	*/
3011	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3012	{
3013	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3014	AssertRCReturn(rc, rc);
3015	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3016	AssertRCReturn(rc, rc);
3017	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3018	AssertRCReturn(rc, rc);
3019	return rc;
3020	}
3021
3022
3023	/**
3024	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3025	* CR0 is partially shared with the host and we have to consider the FPU bits.
3026	*
3027	* @returns VBox status code.
3028	* @param pVM Pointer to the VM.
3029	* @param pVCpu Pointer to the VMCPU.
3030	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3031	* out-of-sync. Make sure to update the required fields
3032	* before using them.
3033	*
3034	* @remarks No-long-jump zone!!!
3035	*/
3036	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3037	{
3038	/*
3039	* Guest CR0.
3040	* Guest FPU.
3041	*/
3042	int rc = VINF_SUCCESS;
3043	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3044	{
3045	Assert(!(pMixedCtx->cr0 >> 32));
3046	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3047	PVM pVM = pVCpu->CTX_SUFF(pVM);
3048
3049	/* The guest's view (read access) of its CR0 is unblemished. */
3050	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3051	AssertRCReturn(rc, rc);
3052	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3053
3054	/* Setup VT-x's view of the guest CR0. */
3055	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3056	if (pVM->hm.s.fNestedPaging)
3057	{
3058	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3059	{
3060	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3061	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3062	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3063	}
3064	else
3065	{
3066	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3067	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3068	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3069	}
3070
3071	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3072	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3073	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3074
3075	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3076	AssertRCReturn(rc, rc);
3077	}
3078	else
3079	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3080
3081	/*
3082	* Guest FPU bits.
3083	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3084	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3085	*/
3086	u32GuestCR0 \|= X86_CR0_NE;
3087	bool fInterceptNM = false;
3088	if (CPUMIsGuestFPUStateActive(pVCpu))
3089	{
3090	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3091	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3092	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3093	}
3094	else
3095	{
3096	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3097	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3098	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3099	}
3100
3101	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3102	bool fInterceptMF = false;
3103	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3104	fInterceptMF = true;
3105
3106	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3107	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3108	{
3109	Assert(PDMVmmDevHeapIsEnabled(pVM));
3110	Assert(pVM->hm.s.vmx.pRealModeTSS);
3111	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3112	fInterceptNM = true;
3113	fInterceptMF = true;
3114	}
3115	else
3116	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3117
3118	if (fInterceptNM)
3119	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3120	else
3121	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3122
3123	if (fInterceptMF)
3124	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3125	else
3126	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3127
3128	/* Additional intercepts for debugging, define these yourself explicitly. */
3129	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3130	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3131	\| RT_BIT(X86_XCPT_BP)
3132	\| RT_BIT(X86_XCPT_DB)
3133	\| RT_BIT(X86_XCPT_DE)
3134	\| RT_BIT(X86_XCPT_NM)
3135	\| RT_BIT(X86_XCPT_UD)
3136	\| RT_BIT(X86_XCPT_NP)
3137	\| RT_BIT(X86_XCPT_SS)
3138	\| RT_BIT(X86_XCPT_GP)
3139	\| RT_BIT(X86_XCPT_PF)
3140	\| RT_BIT(X86_XCPT_MF)
3141	;
3142	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3143	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3144	#endif
3145
3146	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3147
3148	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3149	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3150	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3151	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3152	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3153	else
3154	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3155
3156	u32GuestCR0 \|= uSetCR0;
3157	u32GuestCR0 &= uZapCR0;
3158	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3159
3160	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3161	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3162	AssertRCReturn(rc, rc);
3163	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3164	AssertRCReturn(rc, rc);
3165	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3166
3167	/*
3168	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3169	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3170	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3171	*/
3172	uint32_t u32CR0Mask = 0;
3173	u32CR0Mask = X86_CR0_PE
3174	\| X86_CR0_NE
3175	\| X86_CR0_WP
3176	\| X86_CR0_PG
3177	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3178	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3179	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3180
3181	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3182	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3183	* and @bugref{6944}. */
3184	#if 0
3185	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3186	u32CR0Mask &= ~X86_CR0_PE;
3187	#endif
3188	if (pVM->hm.s.fNestedPaging)
3189	u32CR0Mask &= ~X86_CR0_WP;
3190
3191	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3192	if (fInterceptNM)
3193	{
3194	u32CR0Mask \|= X86_CR0_TS
3195	\| X86_CR0_MP;
3196	}
3197
3198	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3199	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3200	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3201	AssertRCReturn(rc, rc);
3202	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3203
3204	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3205	}
3206	return rc;
3207	}
3208
3209
3210	/**
3211	* Loads the guest control registers (CR3, CR4) into the guest-state area
3212	* in the VMCS.
3213	*
3214	* @returns VBox status code.
3215	* @param pVM Pointer to the VM.
3216	* @param pVCpu Pointer to the VMCPU.
3217	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3218	* out-of-sync. Make sure to update the required fields
3219	* before using them.
3220	*
3221	* @remarks No-long-jump zone!!!
3222	*/
3223	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3224	{
3225	int rc = VINF_SUCCESS;
3226	PVM pVM = pVCpu->CTX_SUFF(pVM);
3227
3228	/*
3229	* Guest CR2.
3230	* It's always loaded in the assembler code. Nothing to do here.
3231	*/
3232
3233	/*
3234	* Guest CR3.
3235	*/
3236	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3237	{
3238	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3239	if (pVM->hm.s.fNestedPaging)
3240	{
3241	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3242
3243	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3244	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3245	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3246	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3247
3248	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3249	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3250	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3251
3252	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3253	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3254	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3255	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3256
3257	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3258	AssertRCReturn(rc, rc);
3259	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3260
3261	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3262	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3263	{
3264	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3265	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3266	{
3267	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3268	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3269	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3270	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3271	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3272	}
3273
3274	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3275	have Unrestricted Execution to handle the guest when it's not using paging. */
3276	GCPhysGuestCR3 = pMixedCtx->cr3;
3277	}
3278	else
3279	{
3280	/*
3281	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3282	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3283	* EPT takes care of translating it to host-physical addresses.
3284	*/
3285	RTGCPHYS GCPhys;
3286	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3287	Assert(PDMVmmDevHeapIsEnabled(pVM));
3288
3289	/* We obtain it here every time as the guest could have relocated this PCI region. */
3290	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3291	AssertRCReturn(rc, rc);
3292
3293	GCPhysGuestCR3 = GCPhys;
3294	}
3295
3296	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3297	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3298	}
3299	else
3300	{
3301	/* Non-nested paging case, just use the hypervisor's CR3. */
3302	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3303
3304	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3305	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3306	}
3307	AssertRCReturn(rc, rc);
3308
3309	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3310	}
3311
3312	/*
3313	* Guest CR4.
3314	*/
3315	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3316	{
3317	Assert(!(pMixedCtx->cr4 >> 32));
3318	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3319
3320	/* The guest's view of its CR4 is unblemished. */
3321	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3322	AssertRCReturn(rc, rc);
3323	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3324
3325	/* Setup VT-x's view of the guest CR4. */
3326	/*
3327	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3328	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3329	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3330	*/
3331	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3332	{
3333	Assert(pVM->hm.s.vmx.pRealModeTSS);
3334	Assert(PDMVmmDevHeapIsEnabled(pVM));
3335	u32GuestCR4 &= ~X86_CR4_VME;
3336	}
3337
3338	if (pVM->hm.s.fNestedPaging)
3339	{
3340	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3341	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3342	{
3343	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3344	u32GuestCR4 \|= X86_CR4_PSE;
3345	/* Our identity mapping is a 32 bits page directory. */
3346	u32GuestCR4 &= ~X86_CR4_PAE;
3347	}
3348	/* else use guest CR4.*/
3349	}
3350	else
3351	{
3352	/*
3353	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3354	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3355	*/
3356	switch (pVCpu->hm.s.enmShadowMode)
3357	{
3358	case PGMMODE_REAL: /* Real-mode. */
3359	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3360	case PGMMODE_32_BIT: /* 32-bit paging. */
3361	{
3362	u32GuestCR4 &= ~X86_CR4_PAE;
3363	break;
3364	}
3365
3366	case PGMMODE_PAE: /* PAE paging. */
3367	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3368	{
3369	u32GuestCR4 \|= X86_CR4_PAE;
3370	break;
3371	}
3372
3373	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3374	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3375	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3376	break;
3377	#endif
3378	default:
3379	AssertFailed();
3380	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3381	}
3382	}
3383
3384	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3385	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3386	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3387	u32GuestCR4 \|= uSetCR4;
3388	u32GuestCR4 &= uZapCR4;
3389
3390	/* Write VT-x's view of the guest CR4 into the VMCS. */
3391	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3392	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3393	AssertRCReturn(rc, rc);
3394
3395	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3396	uint32_t u32CR4Mask = 0;
3397	u32CR4Mask = X86_CR4_VME
3398	\| X86_CR4_PAE
3399	\| X86_CR4_PGE
3400	\| X86_CR4_PSE
3401	\| X86_CR4_VMXE;
3402	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3403	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3404	AssertRCReturn(rc, rc);
3405
3406	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3407	}
3408	return rc;
3409	}
3410
3411
3412	/**
3413	* Loads the guest debug registers into the guest-state area in the VMCS.
3414	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3415	*
3416	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3417	*
3418	* @returns VBox status code.
3419	* @param pVCpu Pointer to the VMCPU.
3420	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3421	* out-of-sync. Make sure to update the required fields
3422	* before using them.
3423	*
3424	* @remarks No-long-jump zone!!!
3425	*/
3426	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3427	{
3428	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3429	return VINF_SUCCESS;
3430
3431	#ifdef VBOX_STRICT
3432	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3433	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3434	{
3435	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3436	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3437	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3438	}
3439	#endif
3440
3441	int rc;
3442	PVM pVM = pVCpu->CTX_SUFF(pVM);
3443	bool fInterceptDB = false;
3444	bool fInterceptMovDRx = false;
3445	if (pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu))
3446	{
3447	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3448	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3449	{
3450	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3451	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3452	AssertRCReturn(rc, rc);
3453	Assert(fInterceptDB == false);
3454	}
3455	else
3456	{
3457	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3458	pVCpu->hm.s.fClearTrapFlag = true;
3459	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3460	fInterceptDB = true;
3461	}
3462	}
3463
3464	if (fInterceptDB \|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3465	{
3466	/*
3467	* Use the combined guest and host DRx values found in the hypervisor
3468	* register set because the debugger has breakpoints active or someone
3469	* is single stepping on the host side without a monitor trap flag.
3470	*
3471	* Note! DBGF expects a clean DR6 state before executing guest code.
3472	*/
3473	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3474	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3475	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3476	{
3477	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3478	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3479	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3480	}
3481	else
3482	#endif
3483	if (!CPUMIsHyperDebugStateActive(pVCpu))
3484	{
3485	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3486	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3487	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3488	}
3489
3490	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3491	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3492	AssertRCReturn(rc, rc);
3493
3494	pVCpu->hm.s.fUsingHyperDR7 = true;
3495	fInterceptDB = true;
3496	fInterceptMovDRx = true;
3497	}
3498	else
3499	{
3500	/*
3501	* If the guest has enabled debug registers, we need to load them prior to
3502	* executing guest code so they'll trigger at the right time.
3503	*/
3504	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3505	{
3506	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3507	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3508	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3509	{
3510	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3511	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3512	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3513	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3514	}
3515	else
3516	#endif
3517	if (CPUMIsGuestDebugStateActive(pVCpu))
3518	{
3519	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3520	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3521	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3522	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3523	}
3524	}
3525	/*
3526	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3527	* must intercept #DB in order to maintain a correct DR6 guest value.
3528	*/
3529	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3530	else if ( ( CPUMIsGuestInLongModeEx(pMixedCtx)
3531	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3532	\|\| !CPUMIsGuestDebugStateActive(pVCpu))
3533	#else
3534	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3535	#endif
3536	{
3537	fInterceptMovDRx = true;
3538	fInterceptDB = true;
3539	}
3540
3541	/* Update guest DR7. */
3542	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3543	AssertRCReturn(rc, rc);
3544
3545	pVCpu->hm.s.fUsingHyperDR7 = false;
3546	}
3547
3548	/*
3549	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3550	*/
3551	if (fInterceptDB)
3552	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3553	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3554	{
3555	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3556	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3557	#endif
3558	}
3559	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3560	AssertRCReturn(rc, rc);
3561
3562	/*
3563	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3564	*/
3565	if (fInterceptMovDRx)
3566	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3567	else
3568	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3569	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3570	AssertRCReturn(rc, rc);
3571
3572	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3573	return VINF_SUCCESS;
3574	}
3575
3576
3577	#ifdef VBOX_STRICT
3578	/**
3579	* Strict function to validate segment registers.
3580	*
3581	* @remarks ASSUMES CR0 is up to date.
3582	*/
3583	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3584	{
3585	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3586	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3587	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3588	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3589	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3590	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3591	{
3592	/* Protected mode checks */
3593	/* CS */
3594	Assert(pCtx->cs.Attr.n.u1Present);
3595	Assert(!(pCtx->cs.Attr.u & 0xf00));
3596	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3597	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3598	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3599	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3600	\|\| (pCtx->cs.Attr.n.u1Granularity));
3601	/* CS cannot be loaded with NULL in protected mode. */
3602	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3603	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3604	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3605	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3606	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3607	else
3608	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3609	/* SS */
3610	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3611	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3612	if ( !(pCtx->cr0 & X86_CR0_PE)
3613	\|\| pCtx->cs.Attr.n.u4Type == 3)
3614	{
3615	Assert(!pCtx->ss.Attr.n.u2Dpl);
3616	}
3617	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3618	{
3619	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3620	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3621	Assert(pCtx->ss.Attr.n.u1Present);
3622	Assert(!(pCtx->ss.Attr.u & 0xf00));
3623	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3624	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3625	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3626	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3627	\|\| (pCtx->ss.Attr.n.u1Granularity));
3628	}
3629	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3630	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3631	{
3632	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3633	Assert(pCtx->ds.Attr.n.u1Present);
3634	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3635	Assert(!(pCtx->ds.Attr.u & 0xf00));
3636	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3637	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3638	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3639	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3640	\|\| (pCtx->ds.Attr.n.u1Granularity));
3641	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3642	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3643	}
3644	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3645	{
3646	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3647	Assert(pCtx->es.Attr.n.u1Present);
3648	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3649	Assert(!(pCtx->es.Attr.u & 0xf00));
3650	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3651	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3652	\|\| !(pCtx->es.Attr.n.u1Granularity));
3653	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3654	\|\| (pCtx->es.Attr.n.u1Granularity));
3655	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3656	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3657	}
3658	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3659	{
3660	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3661	Assert(pCtx->fs.Attr.n.u1Present);
3662	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3663	Assert(!(pCtx->fs.Attr.u & 0xf00));
3664	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3665	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3666	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3667	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3668	\|\| (pCtx->fs.Attr.n.u1Granularity));
3669	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3670	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3671	}
3672	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3673	{
3674	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3675	Assert(pCtx->gs.Attr.n.u1Present);
3676	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3677	Assert(!(pCtx->gs.Attr.u & 0xf00));
3678	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3679	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3680	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3681	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3682	\|\| (pCtx->gs.Attr.n.u1Granularity));
3683	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3684	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3685	}
3686	/* 64-bit capable CPUs. */
3687	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3688	Assert(!(pCtx->cs.u64Base >> 32));
3689	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3690	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3691	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3692	# endif
3693	}
3694	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3695	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3696	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3697	{
3698	/* Real and v86 mode checks. */
3699	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3700	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3701	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3702	{
3703	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3704	}
3705	else
3706	{
3707	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3708	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3709	}
3710
3711	/* CS */
3712	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3713	Assert(pCtx->cs.u32Limit == 0xffff);
3714	Assert(u32CSAttr == 0xf3);
3715	/* SS */
3716	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3717	Assert(pCtx->ss.u32Limit == 0xffff);
3718	Assert(u32SSAttr == 0xf3);
3719	/* DS */
3720	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3721	Assert(pCtx->ds.u32Limit == 0xffff);
3722	Assert(u32DSAttr == 0xf3);
3723	/* ES */
3724	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3725	Assert(pCtx->es.u32Limit == 0xffff);
3726	Assert(u32ESAttr == 0xf3);
3727	/* FS */
3728	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3729	Assert(pCtx->fs.u32Limit == 0xffff);
3730	Assert(u32FSAttr == 0xf3);
3731	/* GS */
3732	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3733	Assert(pCtx->gs.u32Limit == 0xffff);
3734	Assert(u32GSAttr == 0xf3);
3735	/* 64-bit capable CPUs. */
3736	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3737	Assert(!(pCtx->cs.u64Base >> 32));
3738	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3739	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3740	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3741	# endif
3742	}
3743	}
3744	#endif /* VBOX_STRICT */
3745
3746
3747	/**
3748	* Writes a guest segment register into the guest-state area in the VMCS.
3749	*
3750	* @returns VBox status code.
3751	* @param pVCpu Pointer to the VMCPU.
3752	* @param idxSel Index of the selector in the VMCS.
3753	* @param idxLimit Index of the segment limit in the VMCS.
3754	* @param idxBase Index of the segment base in the VMCS.
3755	* @param idxAccess Index of the access rights of the segment in the VMCS.
3756	* @param pSelReg Pointer to the segment selector.
3757	* @param pCtx Pointer to the guest-CPU context.
3758	*
3759	* @remarks No-long-jump zone!!!
3760	*/
3761	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3762	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3763	{
3764	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3765	AssertRCReturn(rc, rc);
3766	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3767	AssertRCReturn(rc, rc);
3768	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3769	AssertRCReturn(rc, rc);
3770
3771	uint32_t u32Access = pSelReg->Attr.u;
3772	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3773	{
3774	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3775	u32Access = 0xf3;
3776	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3777	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3778	}
3779	else
3780	{
3781	/*
3782	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3783	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3784	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3785	* loaded in protected-mode have their attribute as 0.
3786	*/
3787	if (!u32Access)
3788	u32Access = X86DESCATTR_UNUSABLE;
3789	}
3790
3791	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3792	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3793	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3794
3795	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3796	AssertRCReturn(rc, rc);
3797	return rc;
3798	}
3799
3800
3801	/**
3802	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3803	* into the guest-state area in the VMCS.
3804	*
3805	* @returns VBox status code.
3806	* @param pVM Pointer to the VM.
3807	* @param pVCPU Pointer to the VMCPU.
3808	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3809	* out-of-sync. Make sure to update the required fields
3810	* before using them.
3811	*
3812	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
3813	* @remarks No-long-jump zone!!!
3814	*/
3815	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3816	{
3817	int rc = VERR_INTERNAL_ERROR_5;
3818	PVM pVM = pVCpu->CTX_SUFF(pVM);
3819
3820	/*
3821	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3822	*/
3823	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
3824	{
3825	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3826	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3827	{
3828	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3829	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3830	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3831	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3832	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3833	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3834	}
3835
3836	#ifdef VBOX_WITH_REM
3837	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3838	{
3839	Assert(pVM->hm.s.vmx.pRealModeTSS);
3840	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3841	if ( pVCpu->hm.s.vmx.fWasInRealMode
3842	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3843	{
3844	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3845	in real-mode (e.g. OpenBSD 4.0) */
3846	REMFlushTBs(pVM);
3847	Log4(("Load: Switch to protected mode detected!\n"));
3848	pVCpu->hm.s.vmx.fWasInRealMode = false;
3849	}
3850	}
3851	#endif
3852	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3853	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3854	AssertRCReturn(rc, rc);
3855	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3856	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3857	AssertRCReturn(rc, rc);
3858	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3859	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3860	AssertRCReturn(rc, rc);
3861	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3862	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3863	AssertRCReturn(rc, rc);
3864	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3865	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3866	AssertRCReturn(rc, rc);
3867	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3868	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3869	AssertRCReturn(rc, rc);
3870
3871	#ifdef VBOX_STRICT
3872	/* Validate. */
3873	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3874	#endif
3875
3876	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
3877	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3878	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3879	}
3880
3881	/*
3882	* Guest TR.
3883	*/
3884	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
3885	{
3886	/*
3887	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3888	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3889	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3890	*/
3891	uint16_t u16Sel = 0;
3892	uint32_t u32Limit = 0;
3893	uint64_t u64Base = 0;
3894	uint32_t u32AccessRights = 0;
3895
3896	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3897	{
3898	u16Sel = pMixedCtx->tr.Sel;
3899	u32Limit = pMixedCtx->tr.u32Limit;
3900	u64Base = pMixedCtx->tr.u64Base;
3901	u32AccessRights = pMixedCtx->tr.Attr.u;
3902	}
3903	else
3904	{
3905	Assert(pVM->hm.s.vmx.pRealModeTSS);
3906	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3907
3908	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3909	RTGCPHYS GCPhys;
3910	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3911	AssertRCReturn(rc, rc);
3912
3913	X86DESCATTR DescAttr;
3914	DescAttr.u = 0;
3915	DescAttr.n.u1Present = 1;
3916	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3917
3918	u16Sel = 0;
3919	u32Limit = HM_VTX_TSS_SIZE;
3920	u64Base = GCPhys; /* in real-mode phys = virt. */
3921	u32AccessRights = DescAttr.u;
3922	}
3923
3924	/* Validate. */
3925	Assert(!(u16Sel & RT_BIT(2)));
3926	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3927	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3928	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3929	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3930	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3931	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3932	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3933	Assert( (u32Limit & 0xfff) == 0xfff
3934	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3935	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3936	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3937
3938	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3939	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3940	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3941	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3942
3943	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
3944	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3945	}
3946
3947	/*
3948	* Guest GDTR.
3949	*/
3950	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
3951	{
3952	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3953	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3954
3955	/* Validate. */
3956	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3957
3958	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
3959	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3960	}
3961
3962	/*
3963	* Guest LDTR.
3964	*/
3965	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
3966	{
3967	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3968	uint32_t u32Access = 0;
3969	if (!pMixedCtx->ldtr.Attr.u)
3970	u32Access = X86DESCATTR_UNUSABLE;
3971	else
3972	u32Access = pMixedCtx->ldtr.Attr.u;
3973
3974	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3975	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3976	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3977	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3978
3979	/* Validate. */
3980	if (!(u32Access & X86DESCATTR_UNUSABLE))
3981	{
3982	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3983	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3984	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3985	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3986	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3987	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3988	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3989	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3990	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3991	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3992	}
3993
3994	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
3995	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3996	}
3997
3998	/*
3999	* Guest IDTR.
4000	*/
4001	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4002	{
4003	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4004	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4005
4006	/* Validate. */
4007	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4008
4009	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4010	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4011	}
4012
4013	return VINF_SUCCESS;
4014	}
4015
4016
4017	/**
4018	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4019	* areas. These MSRs will automatically be loaded to the host CPU on every
4020	* successful VM entry and stored from the host CPU on every successful VM exit.
4021	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4022	*
4023	* @returns VBox status code.
4024	* @param pVCpu Pointer to the VMCPU.
4025	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4026	* out-of-sync. Make sure to update the required fields
4027	* before using them.
4028	*
4029	* @remarks No-long-jump zone!!!
4030	*/
4031	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4032	{
4033	AssertPtr(pVCpu);
4034	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4035
4036	/*
4037	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
4038	*/
4039	int rc = VINF_SUCCESS;
4040	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4041	{
4042	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
4043	PVM pVM = pVCpu->CTX_SUFF(pVM);
4044	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4045	uint32_t cGuestMsrs = 0;
4046
4047	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
4048	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
4049	* when the guest really is in 64-bit mode. */
4050	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
4051	if (fSupportsLongMode)
4052	{
4053	pGuestMsr->u32Msr = MSR_K8_LSTAR;
4054	pGuestMsr->u32Reserved = 0;
4055	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
4056	pGuestMsr++; cGuestMsrs++;
4057	pGuestMsr->u32Msr = MSR_K6_STAR;
4058	pGuestMsr->u32Reserved = 0;
4059	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
4060	pGuestMsr++; cGuestMsrs++;
4061	pGuestMsr->u32Msr = MSR_K8_SF_MASK;
4062	pGuestMsr->u32Reserved = 0;
4063	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
4064	pGuestMsr++; cGuestMsrs++;
4065	pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
4066	pGuestMsr->u32Reserved = 0;
4067	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
4068	pGuestMsr++; cGuestMsrs++;
4069	}
4070
4071	/*
4072	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
4073	* load the guest's copy always (since the MSR bitmap allows passthru unconditionally).
4074	*/
4075	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
4076	{
4077	pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
4078	pGuestMsr->u32Reserved = 0;
4079	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
4080	AssertRCReturn(rc, rc);
4081	pGuestMsr++; cGuestMsrs++;
4082	}
4083
4084	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
4085	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
4086	{
4087	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
4088	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
4089	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4090	}
4091
4092	/* Update the VCPU's copy of the guest MSR count. */
4093	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
4094	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4095	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4096	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
4097
4098	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4099	}
4100
4101	/*
4102	* Guest Sysenter MSRs.
4103	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4104	* VM-exits on WRMSRs for these MSRs.
4105	*/
4106	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4107	{
4108	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4109	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4110	}
4111
4112	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4113	{
4114	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4115	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4116	}
4117
4118	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4119	{
4120	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4121	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4122	}
4123
4124	return rc;
4125	}
4126
4127
4128	/**
4129	* Loads the guest activity state into the guest-state area in the VMCS.
4130	*
4131	* @returns VBox status code.
4132	* @param pVCpu Pointer to the VMCPU.
4133	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4134	* out-of-sync. Make sure to update the required fields
4135	* before using them.
4136	*
4137	* @remarks No-long-jump zone!!!
4138	*/
4139	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4140	{
4141	/** @todo See if we can make use of other states, e.g.
4142	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4143	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4144	{
4145	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4146	AssertRCReturn(rc, rc);
4147
4148	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4149	}
4150	return VINF_SUCCESS;
4151	}
4152
4153
4154	/**
4155	* Sets up the appropriate function to run guest code.
4156	*
4157	* @returns VBox status code.
4158	* @param pVCpu Pointer to the VMCPU.
4159	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4160	* out-of-sync. Make sure to update the required fields
4161	* before using them.
4162	*
4163	* @remarks No-long-jump zone!!!
4164	*/
4165	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4166	{
4167	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4168	{
4169	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4170	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4171	#endif
4172	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4173	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4174	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4175	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4176	{
4177	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4178	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4179	}
4180	#else
4181	/* 64-bit host or hybrid host. */
4182	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4183	#endif
4184	}
4185	else
4186	{
4187	/* Guest is not in long mode, use the 32-bit handler. */
4188	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4189	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4190	{
4191	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4192	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4193	}
4194	#else
4195	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4196	#endif
4197	}
4198	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4199	return VINF_SUCCESS;
4200	}
4201
4202
4203	/**
4204	* Wrapper for running the guest code in VT-x.
4205	*
4206	* @returns VBox strict status code.
4207	* @param pVM Pointer to the VM.
4208	* @param pVCpu Pointer to the VMCPU.
4209	* @param pCtx Pointer to the guest-CPU context.
4210	*
4211	* @remarks No-long-jump zone!!!
4212	*/
4213	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4214	{
4215	/*
4216	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4217	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4218	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4219	*/
4220	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4221	/** @todo Add stats for resume vs launch. */
4222	#ifdef VBOX_WITH_KERNEL_USING_XMM
4223	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4224	#else
4225	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4226	#endif
4227	}
4228
4229
4230	/**
4231	* Reports world-switch error and dumps some useful debug info.
4232	*
4233	* @param pVM Pointer to the VM.
4234	* @param pVCpu Pointer to the VMCPU.
4235	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4236	* @param pCtx Pointer to the guest-CPU context.
4237	* @param pVmxTransient Pointer to the VMX transient structure (only
4238	* exitReason updated).
4239	*/
4240	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4241	{
4242	Assert(pVM);
4243	Assert(pVCpu);
4244	Assert(pCtx);
4245	Assert(pVmxTransient);
4246	HMVMX_ASSERT_PREEMPT_SAFE();
4247
4248	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4249	switch (rcVMRun)
4250	{
4251	case VERR_VMX_INVALID_VMXON_PTR:
4252	AssertFailed();
4253	break;
4254	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4255	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4256	{
4257	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4258	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4259	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4260	AssertRC(rc);
4261
4262	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4263	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4264	Cannot do it here as we may have been long preempted. */
4265
4266	#ifdef VBOX_STRICT
4267	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4268	pVmxTransient->uExitReason));
4269	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4270	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4271	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4272	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4273	else
4274	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4275	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4276	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4277
4278	/* VMX control bits. */
4279	uint32_t u32Val;
4280	uint64_t u64Val;
4281	HMVMXHCUINTREG uHCReg;
4282	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4283	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4284	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4285	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4286	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4287	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4288	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4289	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4290	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4291	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4292	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4293	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4294	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4295	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4296	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4297	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4298	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4299	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4300	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4301	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4302	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4303	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4304	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4305	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4306	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4307	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4308	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4309	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4310	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4311	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4312	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4313	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4314	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4315	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4316	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4317	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4318	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4319	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4320	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4321	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4322	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4323	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4324
4325	/* Guest bits. */
4326	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4327	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4328	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4329	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4330	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4331	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4332	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4333	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4334
4335	/* Host bits. */
4336	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4337	Log4(("Host CR0 %#RHr\n", uHCReg));
4338	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4339	Log4(("Host CR3 %#RHr\n", uHCReg));
4340	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4341	Log4(("Host CR4 %#RHr\n", uHCReg));
4342
4343	RTGDTR HostGdtr;
4344	PCX86DESCHC pDesc;
4345	ASMGetGDTR(&HostGdtr);
4346	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4347	Log4(("Host CS %#08x\n", u32Val));
4348	if (u32Val < HostGdtr.cbGdt)
4349	{
4350	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4351	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4352	}
4353
4354	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4355	Log4(("Host DS %#08x\n", u32Val));
4356	if (u32Val < HostGdtr.cbGdt)
4357	{
4358	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4359	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4360	}
4361
4362	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4363	Log4(("Host ES %#08x\n", u32Val));
4364	if (u32Val < HostGdtr.cbGdt)
4365	{
4366	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4367	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4368	}
4369
4370	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4371	Log4(("Host FS %#08x\n", u32Val));
4372	if (u32Val < HostGdtr.cbGdt)
4373	{
4374	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4375	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4376	}
4377
4378	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4379	Log4(("Host GS %#08x\n", u32Val));
4380	if (u32Val < HostGdtr.cbGdt)
4381	{
4382	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4383	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4384	}
4385
4386	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4387	Log4(("Host SS %#08x\n", u32Val));
4388	if (u32Val < HostGdtr.cbGdt)
4389	{
4390	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4391	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4392	}
4393
4394	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4395	Log4(("Host TR %#08x\n", u32Val));
4396	if (u32Val < HostGdtr.cbGdt)
4397	{
4398	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4399	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4400	}
4401
4402	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4403	Log4(("Host TR Base %#RHv\n", uHCReg));
4404	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4405	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4406	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4407	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4408	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4409	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4410	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4411	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4412	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4413	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4414	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4415	Log4(("Host RSP %#RHv\n", uHCReg));
4416	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4417	Log4(("Host RIP %#RHv\n", uHCReg));
4418	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4419	if (HMVMX_IS_64BIT_HOST_MODE())
4420	{
4421	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4422	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4423	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4424	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4425	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4426	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4427	}
4428	# endif
4429	#endif /* VBOX_STRICT */
4430	break;
4431	}
4432
4433	default:
4434	/* Impossible */
4435	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4436	break;
4437	}
4438	NOREF(pVM);
4439	}
4440
4441
4442	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4443	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4444	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4445	#endif
4446	#ifdef VBOX_STRICT
4447	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4448	{
4449	switch (idxField)
4450	{
4451	case VMX_VMCS_GUEST_RIP:
4452	case VMX_VMCS_GUEST_RSP:
4453	case VMX_VMCS_GUEST_SYSENTER_EIP:
4454	case VMX_VMCS_GUEST_SYSENTER_ESP:
4455	case VMX_VMCS_GUEST_GDTR_BASE:
4456	case VMX_VMCS_GUEST_IDTR_BASE:
4457	case VMX_VMCS_GUEST_CS_BASE:
4458	case VMX_VMCS_GUEST_DS_BASE:
4459	case VMX_VMCS_GUEST_ES_BASE:
4460	case VMX_VMCS_GUEST_FS_BASE:
4461	case VMX_VMCS_GUEST_GS_BASE:
4462	case VMX_VMCS_GUEST_SS_BASE:
4463	case VMX_VMCS_GUEST_LDTR_BASE:
4464	case VMX_VMCS_GUEST_TR_BASE:
4465	case VMX_VMCS_GUEST_CR3:
4466	return true;
4467	}
4468	return false;
4469	}
4470
4471	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4472	{
4473	switch (idxField)
4474	{
4475	/* Read-only fields. */
4476	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4477	return true;
4478	}
4479	/* Remaining readable fields should also be writable. */
4480	return hmR0VmxIsValidWriteField(idxField);
4481	}
4482	#endif /* VBOX_STRICT */
4483
4484
4485	/**
4486	* Executes the specified handler in 64-bit mode.
4487	*
4488	* @returns VBox status code.
4489	* @param pVM Pointer to the VM.
4490	* @param pVCpu Pointer to the VMCPU.
4491	* @param pCtx Pointer to the guest CPU context.
4492	* @param enmOp The operation to perform.
4493	* @param cbParam Number of parameters.
4494	* @param paParam Array of 32-bit parameters.
4495	*/
4496	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4497	uint32_t *paParam)
4498	{
4499	int rc, rc2;
4500	PHMGLOBALCPUINFO pCpu;
4501	RTHCPHYS HCPhysCpuPage;
4502	RTCCUINTREG uOldEflags;
4503
4504	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4505	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4506	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4507	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4508
4509	#ifdef VBOX_STRICT
4510	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4511	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4512
4513	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4514	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4515	#endif
4516
4517	/* Disable interrupts. */
4518	uOldEflags = ASMIntDisableFlags();
4519
4520	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4521	RTCPUID idHostCpu = RTMpCpuId();
4522	CPUMR0SetLApic(pVCpu, idHostCpu);
4523	#endif
4524
4525	pCpu = HMR0GetCurrentCpu();
4526	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4527
4528	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4529	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4530
4531	/* Leave VMX Root Mode. */
4532	VMXDisable();
4533
4534	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4535
4536	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4537	CPUMSetHyperEIP(pVCpu, enmOp);
4538	for (int i = (int)cbParam - 1; i >= 0; i--)
4539	CPUMPushHyper(pVCpu, paParam[i]);
4540
4541	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4542
4543	/* Call the switcher. */
4544	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4545	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4546
4547	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4548	/* Make sure the VMX instructions don't cause #UD faults. */
4549	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4550
4551	/* Re-enter VMX Root Mode */
4552	rc2 = VMXEnable(HCPhysCpuPage);
4553	if (RT_FAILURE(rc2))
4554	{
4555	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4556	ASMSetFlags(uOldEflags);
4557	return rc2;
4558	}
4559
4560	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4561	AssertRC(rc2);
4562	Assert(!(ASMGetFlags() & X86_EFL_IF));
4563	ASMSetFlags(uOldEflags);
4564	return rc;
4565	}
4566
4567
4568	/**
4569	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4570	* supporting 64-bit guests.
4571	*
4572	* @returns VBox status code.
4573	* @param fResume Whether to VMLAUNCH or VMRESUME.
4574	* @param pCtx Pointer to the guest-CPU context.
4575	* @param pCache Pointer to the VMCS cache.
4576	* @param pVM Pointer to the VM.
4577	* @param pVCpu Pointer to the VMCPU.
4578	*/
4579	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4580	{
4581	uint32_t aParam[6];
4582	PHMGLOBALCPUINFO pCpu = NULL;
4583	RTHCPHYS HCPhysCpuPage = 0;
4584	int rc = VERR_INTERNAL_ERROR_5;
4585
4586	pCpu = HMR0GetCurrentCpu();
4587	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4588
4589	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4590	pCache->uPos = 1;
4591	pCache->interPD = PGMGetInterPaeCR3(pVM);
4592	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4593	#endif
4594
4595	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4596	pCache->TestIn.HCPhysCpuPage = 0;
4597	pCache->TestIn.HCPhysVmcs = 0;
4598	pCache->TestIn.pCache = 0;
4599	pCache->TestOut.HCPhysVmcs = 0;
4600	pCache->TestOut.pCache = 0;
4601	pCache->TestOut.pCtx = 0;
4602	pCache->TestOut.eflags = 0;
4603	#endif
4604
4605	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4606	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4607	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4608	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4609	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4610	aParam[5] = 0;
4611
4612	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4613	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4614	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4615	#endif
4616	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4617
4618	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4619	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4620	Assert(pCtx->dr[4] == 10);
4621	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4622	#endif
4623
4624	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4625	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4626	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4627	pVCpu->hm.s.vmx.HCPhysVmcs));
4628	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4629	pCache->TestOut.HCPhysVmcs));
4630	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4631	pCache->TestOut.pCache));
4632	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4633	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4634	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4635	pCache->TestOut.pCtx));
4636	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4637	#endif
4638	return rc;
4639	}
4640
4641
4642	/**
4643	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4644	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4645	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4646	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4647	*
4648	* @returns VBox status code.
4649	* @param pVM Pointer to the VM.
4650	* @param pVCpu Pointer to the VMCPU.
4651	*/
4652	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4653	{
4654	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4655	{ \
4656	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4657	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4658	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4659	++cReadFields; \
4660	}
4661
4662	AssertPtr(pVM);
4663	AssertPtr(pVCpu);
4664	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4665	uint32_t cReadFields = 0;
4666
4667	/*
4668	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4669	* and serve to indicate exceptions to the rules.
4670	*/
4671
4672	/* Guest-natural selector base fields. */
4673	#if 0
4674	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4675	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4676	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4677	#endif
4678	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4679	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4680	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4681	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4682	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4683	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4684	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4685	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4686	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4687	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4688	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4689	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4690	#if 0
4691	/* Unused natural width guest-state fields. */
4692	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4693	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4694	#endif
4695	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4696	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4697
4698	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4699	#if 0
4700	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4701	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4702	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4703	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4704	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4705	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4706	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4707	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4708	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4709	#endif
4710
4711	/* Natural width guest-state fields. */
4712	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4713	#if 0
4714	/* Currently unused field. */
4715	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4716	#endif
4717
4718	if (pVM->hm.s.fNestedPaging)
4719	{
4720	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4721	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4722	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4723	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4724	}
4725	else
4726	{
4727	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4728	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4729	}
4730
4731	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4732	return VINF_SUCCESS;
4733	}
4734
4735
4736	/**
4737	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4738	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4739	* darwin, running 64-bit guests).
4740	*
4741	* @returns VBox status code.
4742	* @param pVCpu Pointer to the VMCPU.
4743	* @param idxField The VMCS field encoding.
4744	* @param u64Val 16, 32 or 64 bits value.
4745	*/
4746	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4747	{
4748	int rc;
4749	switch (idxField)
4750	{
4751	/*
4752	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4753	*/
4754	/* 64-bit Control fields. */
4755	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4756	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4757	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4758	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4759	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4760	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4761	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4762	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4763	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4764	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4765	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4766	case VMX_VMCS64_CTRL_EPTP_FULL:
4767	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4768	/* 64-bit Guest-state fields. */
4769	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4770	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4771	case VMX_VMCS64_GUEST_PAT_FULL:
4772	case VMX_VMCS64_GUEST_EFER_FULL:
4773	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4774	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4775	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4776	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4777	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4778	/* 64-bit Host-state fields. */
4779	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4780	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4781	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4782	{
4783	rc = VMXWriteVmcs32(idxField, u64Val);
4784	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4785	break;
4786	}
4787
4788	/*
4789	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4790	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4791	*/
4792	/* Natural-width Guest-state fields. */
4793	case VMX_VMCS_GUEST_CR3:
4794	case VMX_VMCS_GUEST_ES_BASE:
4795	case VMX_VMCS_GUEST_CS_BASE:
4796	case VMX_VMCS_GUEST_SS_BASE:
4797	case VMX_VMCS_GUEST_DS_BASE:
4798	case VMX_VMCS_GUEST_FS_BASE:
4799	case VMX_VMCS_GUEST_GS_BASE:
4800	case VMX_VMCS_GUEST_LDTR_BASE:
4801	case VMX_VMCS_GUEST_TR_BASE:
4802	case VMX_VMCS_GUEST_GDTR_BASE:
4803	case VMX_VMCS_GUEST_IDTR_BASE:
4804	case VMX_VMCS_GUEST_RSP:
4805	case VMX_VMCS_GUEST_RIP:
4806	case VMX_VMCS_GUEST_SYSENTER_ESP:
4807	case VMX_VMCS_GUEST_SYSENTER_EIP:
4808	{
4809	if (!(u64Val >> 32))
4810	{
4811	/* If this field is 64-bit, VT-x will zero out the top bits. */
4812	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4813	}
4814	else
4815	{
4816	/* Assert that only the 32->64 switcher case should ever come here. */
4817	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4818	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4819	}
4820	break;
4821	}
4822
4823	default:
4824	{
4825	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4826	rc = VERR_INVALID_PARAMETER;
4827	break;
4828	}
4829	}
4830	AssertRCReturn(rc, rc);
4831	return rc;
4832	}
4833
4834
4835	/**
4836	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4837	* hosts (except darwin) for 64-bit guests.
4838	*
4839	* @param pVCpu Pointer to the VMCPU.
4840	* @param idxField The VMCS field encoding.
4841	* @param u64Val 16, 32 or 64 bits value.
4842	*/
4843	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4844	{
4845	AssertPtr(pVCpu);
4846	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4847
4848	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4849	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4850
4851	/* Make sure there are no duplicates. */
4852	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4853	{
4854	if (pCache->Write.aField[i] == idxField)
4855	{
4856	pCache->Write.aFieldVal[i] = u64Val;
4857	return VINF_SUCCESS;
4858	}
4859	}
4860
4861	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4862	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4863	pCache->Write.cValidEntries++;
4864	return VINF_SUCCESS;
4865	}
4866
4867	/* Enable later when the assembly code uses these as callbacks. */
4868	#if 0
4869	/*
4870	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4871	*
4872	* @param pVCpu Pointer to the VMCPU.
4873	* @param pCache Pointer to the VMCS cache.
4874	*
4875	* @remarks No-long-jump zone!!!
4876	*/
4877	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4878	{
4879	AssertPtr(pCache);
4880	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4881	{
4882	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4883	AssertRC(rc);
4884	}
4885	pCache->Write.cValidEntries = 0;
4886	}
4887
4888
4889	/**
4890	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4891	*
4892	* @param pVCpu Pointer to the VMCPU.
4893	* @param pCache Pointer to the VMCS cache.
4894	*
4895	* @remarks No-long-jump zone!!!
4896	*/
4897	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4898	{
4899	AssertPtr(pCache);
4900	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4901	{
4902	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4903	AssertRC(rc);
4904	}
4905	}
4906	#endif
4907	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4908
4909
4910	/**
4911	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4912	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4913	* timer.
4914	*
4915	* @returns VBox status code.
4916	* @param pVCpu Pointer to the VMCPU.
4917	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4918	* out-of-sync. Make sure to update the required fields
4919	* before using them.
4920	* @remarks No-long-jump zone!!!
4921	*/
4922	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4923	{
4924	int rc = VERR_INTERNAL_ERROR_5;
4925	bool fOffsettedTsc = false;
4926	PVM pVM = pVCpu->CTX_SUFF(pVM);
4927	if (pVM->hm.s.vmx.fUsePreemptTimer)
4928	{
4929	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4930
4931	/* Make sure the returned values have sane upper and lower boundaries. */
4932	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4933	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4934	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4935	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4936
4937	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4938	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4939	}
4940	else
4941	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4942
4943	if (fOffsettedTsc)
4944	{
4945	uint64_t u64CurTSC = ASMReadTSC();
4946	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4947	{
4948	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4949	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4950
4951	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4952	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4953	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4954	}
4955	else
4956	{
4957	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4958	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4959	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4960	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4961	}
4962	}
4963	else
4964	{
4965	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4966	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4967	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4968	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4969	}
4970	}
4971
4972
4973	/**
4974	* Determines if an exception is a contributory exception. Contributory
4975	* exceptions are ones which can cause double-faults. Page-fault is
4976	* intentionally not included here as it's a conditional contributory exception.
4977	*
4978	* @returns true if the exception is contributory, false otherwise.
4979	* @param uVector The exception vector.
4980	*/
4981	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4982	{
4983	switch (uVector)
4984	{
4985	case X86_XCPT_GP:
4986	case X86_XCPT_SS:
4987	case X86_XCPT_NP:
4988	case X86_XCPT_TS:
4989	case X86_XCPT_DE:
4990	return true;
4991	default:
4992	break;
4993	}
4994	return false;
4995	}
4996
4997
4998	/**
4999	* Sets an event as a pending event to be injected into the guest.
5000	*
5001	* @param pVCpu Pointer to the VMCPU.
5002	* @param u32IntInfo The VM-entry interruption-information field.
5003	* @param cbInstr The VM-entry instruction length in bytes (for software
5004	* interrupts, exceptions and privileged software
5005	* exceptions).
5006	* @param u32ErrCode The VM-entry exception error code.
5007	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5008	* page-fault.
5009	*
5010	* @remarks Statistics counter assumes this is a guest event being injected or
5011	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5012	* always incremented.
5013	*/
5014	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5015	RTGCUINTPTR GCPtrFaultAddress)
5016	{
5017	Assert(!pVCpu->hm.s.Event.fPending);
5018	pVCpu->hm.s.Event.fPending = true;
5019	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5020	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5021	pVCpu->hm.s.Event.cbInstr = cbInstr;
5022	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5023
5024	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5025	}
5026
5027
5028	/**
5029	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5030	*
5031	* @param pVCpu Pointer to the VMCPU.
5032	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5033	* out-of-sync. Make sure to update the required fields
5034	* before using them.
5035	*/
5036	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5037	{
5038	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5039	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5040	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5041	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5042	}
5043
5044
5045	/**
5046	* Handle a condition that occurred while delivering an event through the guest
5047	* IDT.
5048	*
5049	* @returns VBox status code (informational error codes included).
5050	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5051	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5052	* continue execution of the guest which will delivery the #DF.
5053	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5054	*
5055	* @param pVCpu Pointer to the VMCPU.
5056	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5057	* out-of-sync. Make sure to update the required fields
5058	* before using them.
5059	* @param pVmxTransient Pointer to the VMX transient structure.
5060	*
5061	* @remarks No-long-jump zone!!!
5062	*/
5063	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5064	{
5065	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5066	AssertRCReturn(rc, rc);
5067	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5068	{
5069	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
5070	AssertRCReturn(rc, rc);
5071
5072	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5073	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5074	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5075
5076	typedef enum
5077	{
5078	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5079	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5080	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5081	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5082	} VMXREFLECTXCPT;
5083
5084	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5085	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5086	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5087	{
5088	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5089	{
5090	enmReflect = VMXREFLECTXCPT_XCPT;
5091	#ifdef VBOX_STRICT
5092	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5093	&& uExitVector == X86_XCPT_PF)
5094	{
5095	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5096	}
5097	#endif
5098	if ( uExitVector == X86_XCPT_PF
5099	&& uIdtVector == X86_XCPT_PF)
5100	{
5101	pVmxTransient->fVectoringPF = true;
5102	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5103	}
5104	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5105	&& hmR0VmxIsContributoryXcpt(uExitVector)
5106	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5107	\|\| uIdtVector == X86_XCPT_PF))
5108	{
5109	enmReflect = VMXREFLECTXCPT_DF;
5110	}
5111	else if (uIdtVector == X86_XCPT_DF)
5112	enmReflect = VMXREFLECTXCPT_TF;
5113	}
5114	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5115	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5116	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5117	{
5118	/*
5119	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5120	* (whatever they are) as they reoccur when restarting the instruction.
5121	*/
5122	enmReflect = VMXREFLECTXCPT_XCPT;
5123	}
5124	}
5125	else
5126	{
5127	/*
5128	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5129	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5130	* original exception to the guest after handling the VM-exit.
5131	*/
5132	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5133	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5134	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5135	{
5136	enmReflect = VMXREFLECTXCPT_XCPT;
5137	}
5138	}
5139
5140	switch (enmReflect)
5141	{
5142	case VMXREFLECTXCPT_XCPT:
5143	{
5144	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5145	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5146	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5147
5148	uint32_t u32ErrCode = 0;
5149	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5150	{
5151	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5152	AssertRCReturn(rc, rc);
5153	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5154	}
5155
5156	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5157	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5158	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5159	rc = VINF_SUCCESS;
5160	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5161	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5162
5163	break;
5164	}
5165
5166	case VMXREFLECTXCPT_DF:
5167	{
5168	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5169	rc = VINF_HM_DOUBLE_FAULT;
5170	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5171	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5172
5173	break;
5174	}
5175
5176	case VMXREFLECTXCPT_TF:
5177	{
5178	rc = VINF_EM_RESET;
5179	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5180	uExitVector));
5181	break;
5182	}
5183
5184	default:
5185	Assert(rc == VINF_SUCCESS);
5186	break;
5187	}
5188	}
5189	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5190	return rc;
5191	}
5192
5193
5194	/**
5195	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5196	*
5197	* @returns VBox status code.
5198	* @param pVCpu Pointer to the VMCPU.
5199	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5200	* out-of-sync. Make sure to update the required fields
5201	* before using them.
5202	*
5203	* @remarks No-long-jump zone!!!
5204	*/
5205	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5206	{
5207	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5208	{
5209	uint32_t uVal = 0;
5210	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5211	AssertRCReturn(rc, rc);
5212
5213	uint32_t uShadow = 0;
5214	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5215	AssertRCReturn(rc, rc);
5216
5217	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5218	CPUMSetGuestCR0(pVCpu, uVal);
5219	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5220	}
5221	return VINF_SUCCESS;
5222	}
5223
5224
5225	/**
5226	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5227	*
5228	* @returns VBox status code.
5229	* @param pVCpu Pointer to the VMCPU.
5230	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5231	* out-of-sync. Make sure to update the required fields
5232	* before using them.
5233	*
5234	* @remarks No-long-jump zone!!!
5235	*/
5236	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5237	{
5238	int rc = VINF_SUCCESS;
5239	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5240	{
5241	uint32_t uVal = 0;
5242	uint32_t uShadow = 0;
5243	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5244	AssertRCReturn(rc, rc);
5245	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5246	AssertRCReturn(rc, rc);
5247
5248	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5249	CPUMSetGuestCR4(pVCpu, uVal);
5250	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5251	}
5252	return rc;
5253	}
5254
5255
5256	/**
5257	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5258	*
5259	* @returns VBox status code.
5260	* @param pVCpu Pointer to the VMCPU.
5261	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5262	* out-of-sync. Make sure to update the required fields
5263	* before using them.
5264	*
5265	* @remarks No-long-jump zone!!!
5266	*/
5267	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5268	{
5269	int rc = VINF_SUCCESS;
5270	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5271	{
5272	uint64_t u64Val = 0;
5273	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5274	AssertRCReturn(rc, rc);
5275
5276	pMixedCtx->rip = u64Val;
5277	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5278	}
5279	return rc;
5280	}
5281
5282
5283	/**
5284	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5285	*
5286	* @returns VBox status code.
5287	* @param pVCpu Pointer to the VMCPU.
5288	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5289	* out-of-sync. Make sure to update the required fields
5290	* before using them.
5291	*
5292	* @remarks No-long-jump zone!!!
5293	*/
5294	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5295	{
5296	int rc = VINF_SUCCESS;
5297	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5298	{
5299	uint64_t u64Val = 0;
5300	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5301	AssertRCReturn(rc, rc);
5302
5303	pMixedCtx->rsp = u64Val;
5304	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5305	}
5306	return rc;
5307	}
5308
5309
5310	/**
5311	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5312	*
5313	* @returns VBox status code.
5314	* @param pVCpu Pointer to the VMCPU.
5315	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5316	* out-of-sync. Make sure to update the required fields
5317	* before using them.
5318	*
5319	* @remarks No-long-jump zone!!!
5320	*/
5321	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5322	{
5323	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5324	{
5325	uint32_t uVal = 0;
5326	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5327	AssertRCReturn(rc, rc);
5328
5329	pMixedCtx->eflags.u32 = uVal;
5330	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5331	{
5332	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5333	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5334
5335	pMixedCtx->eflags.Bits.u1VM = 0;
5336	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5337	}
5338
5339	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5340	}
5341	return VINF_SUCCESS;
5342	}
5343
5344
5345	/**
5346	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5347	* guest-CPU context.
5348	*/
5349	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5350	{
5351	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5352	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5353	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5354	return rc;
5355	}
5356
5357
5358	/**
5359	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5360	* from the guest-state area in the VMCS.
5361	*
5362	* @param pVCpu Pointer to the VMCPU.
5363	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5364	* out-of-sync. Make sure to update the required fields
5365	* before using them.
5366	*
5367	* @remarks No-long-jump zone!!!
5368	*/
5369	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5370	{
5371	uint32_t uIntrState = 0;
5372	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5373	AssertRC(rc);
5374
5375	if (!uIntrState)
5376	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5377	else
5378	{
5379	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5380	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5381	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5382	AssertRC(rc);
5383	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5384	AssertRC(rc);
5385
5386	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5387	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5388	}
5389	}
5390
5391
5392	/**
5393	* Saves the guest's activity state.
5394	*
5395	* @returns VBox status code.
5396	* @param pVCpu Pointer to the VMCPU.
5397	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5398	* out-of-sync. Make sure to update the required fields
5399	* before using them.
5400	*
5401	* @remarks No-long-jump zone!!!
5402	*/
5403	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5404	{
5405	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5406	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5407	return VINF_SUCCESS;
5408	}
5409
5410
5411	/**
5412	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5413	* the current VMCS into the guest-CPU context.
5414	*
5415	* @returns VBox status code.
5416	* @param pVCpu Pointer to the VMCPU.
5417	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5418	* out-of-sync. Make sure to update the required fields
5419	* before using them.
5420	*
5421	* @remarks No-long-jump zone!!!
5422	*/
5423	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5424	{
5425	int rc = VINF_SUCCESS;
5426	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5427	{
5428	uint32_t u32Val = 0;
5429	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5430	pMixedCtx->SysEnter.cs = u32Val;
5431	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5432	}
5433
5434	uint64_t u64Val = 0;
5435	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5436	{
5437	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5438	pMixedCtx->SysEnter.eip = u64Val;
5439	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5440	}
5441	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5442	{
5443	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5444	pMixedCtx->SysEnter.esp = u64Val;
5445	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5446	}
5447	return rc;
5448	}
5449
5450
5451	/**
5452	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5453	* context.
5454	*
5455	* @returns VBox status code.
5456	* @param pVCpu Pointer to the VMCPU.
5457	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5458	* out-of-sync. Make sure to update the required fields
5459	* before using them.
5460	*
5461	* @remarks No-long-jump zone!!!
5462	*/
5463	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5464	{
5465	int rc = VINF_SUCCESS;
5466	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5467	{
5468	uint64_t u64Val = 0;
5469	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5470	pMixedCtx->fs.u64Base = u64Val;
5471	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5472	}
5473	return rc;
5474	}
5475
5476
5477	/**
5478	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5479	* context.
5480	*
5481	* @returns VBox status code.
5482	* @param pVCpu Pointer to the VMCPU.
5483	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5484	* out-of-sync. Make sure to update the required fields
5485	* before using them.
5486	*
5487	* @remarks No-long-jump zone!!!
5488	*/
5489	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5490	{
5491	int rc = VINF_SUCCESS;
5492	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5493	{
5494	uint64_t u64Val = 0;
5495	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5496	pMixedCtx->gs.u64Base = u64Val;
5497	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5498	}
5499	return rc;
5500	}
5501
5502
5503	/**
5504	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5505	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5506	* and TSC_AUX.
5507	*
5508	* @returns VBox status code.
5509	* @param pVCpu Pointer to the VMCPU.
5510	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5511	* out-of-sync. Make sure to update the required fields
5512	* before using them.
5513	*
5514	* @remarks No-long-jump zone!!!
5515	*/
5516	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5517	{
5518	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5519	return VINF_SUCCESS;
5520
5521	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5522	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5523	{
5524	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5525	pMsr += i;
5526	switch (pMsr->u32Msr)
5527	{
5528	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5529	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5530	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5531	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5532	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5533	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5534	default:
5535	{
5536	AssertFailed();
5537	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5538	}
5539	}
5540	}
5541	#endif
5542
5543	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5544	return VINF_SUCCESS;
5545	}
5546
5547
5548	/**
5549	* Saves the guest control registers from the current VMCS into the guest-CPU
5550	* context.
5551	*
5552	* @returns VBox status code.
5553	* @param pVCpu Pointer to the VMCPU.
5554	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5555	* out-of-sync. Make sure to update the required fields
5556	* before using them.
5557	*
5558	* @remarks No-long-jump zone!!!
5559	*/
5560	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5561	{
5562	/* Guest CR0. Guest FPU. */
5563	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5564	AssertRCReturn(rc, rc);
5565
5566	/* Guest CR4. */
5567	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5568	AssertRCReturn(rc, rc);
5569
5570	/* Guest CR2 - updated always during the world-switch or in #PF. */
5571	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5572	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5573	{
5574	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5575	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5576
5577	PVM pVM = pVCpu->CTX_SUFF(pVM);
5578	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5579	\|\| ( pVM->hm.s.fNestedPaging
5580	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5581	{
5582	uint64_t u64Val = 0;
5583	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5584	if (pMixedCtx->cr3 != u64Val)
5585	{
5586	CPUMSetGuestCR3(pVCpu, u64Val);
5587	if (VMMRZCallRing3IsEnabled(pVCpu))
5588	{
5589	PGMUpdateCR3(pVCpu, u64Val);
5590	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5591	}
5592	else
5593	{
5594	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5595	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5596	}
5597	}
5598
5599	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5600	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5601	{
5602	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5603	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5604	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5605	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5606
5607	if (VMMRZCallRing3IsEnabled(pVCpu))
5608	{
5609	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5610	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5611	}
5612	else
5613	{
5614	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5615	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5616	}
5617	}
5618	}
5619
5620	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5621	}
5622
5623	/*
5624	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5625	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5626	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5627	*
5628	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5629	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5630	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5631	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
5632	*
5633	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5634	*/
5635	if (VMMRZCallRing3IsEnabled(pVCpu))
5636	{
5637	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5638	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5639
5640	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5641	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5642
5643	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5644	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5645	}
5646
5647	return rc;
5648	}
5649
5650
5651	/**
5652	* Reads a guest segment register from the current VMCS into the guest-CPU
5653	* context.
5654	*
5655	* @returns VBox status code.
5656	* @param pVCpu Pointer to the VMCPU.
5657	* @param idxSel Index of the selector in the VMCS.
5658	* @param idxLimit Index of the segment limit in the VMCS.
5659	* @param idxBase Index of the segment base in the VMCS.
5660	* @param idxAccess Index of the access rights of the segment in the VMCS.
5661	* @param pSelReg Pointer to the segment selector.
5662	*
5663	* @remarks No-long-jump zone!!!
5664	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5665	* macro as that takes care of whether to read from the VMCS cache or
5666	* not.
5667	*/
5668	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5669	PCPUMSELREG pSelReg)
5670	{
5671	uint32_t u32Val = 0;
5672	int rc = VMXReadVmcs32(idxSel, &u32Val);
5673	AssertRCReturn(rc, rc);
5674	pSelReg->Sel = (uint16_t)u32Val;
5675	pSelReg->ValidSel = (uint16_t)u32Val;
5676	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5677
5678	rc = VMXReadVmcs32(idxLimit, &u32Val);
5679	AssertRCReturn(rc, rc);
5680	pSelReg->u32Limit = u32Val;
5681
5682	uint64_t u64Val = 0;
5683	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5684	AssertRCReturn(rc, rc);
5685	pSelReg->u64Base = u64Val;
5686
5687	rc = VMXReadVmcs32(idxAccess, &u32Val);
5688	AssertRCReturn(rc, rc);
5689	pSelReg->Attr.u = u32Val;
5690
5691	/*
5692	* If VT-x marks the segment as unusable, most other bits remain undefined:
5693	* - For CS the L, D and G bits have meaning.
5694	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5695	* - For the remaining data segments no bits are defined.
5696	*
5697	* The present bit and the unusable bit has been observed to be set at the
5698	* same time (the selector was supposed to invalid as we started executing
5699	* a V8086 interrupt in ring-0).
5700	*
5701	* What should be important for the rest of the VBox code that the P bit is
5702	* cleared. Some of the other VBox code recognizes the unusable bit, but
5703	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
5704	* safe side here, we'll strip off P and other bits we don't care about. If
5705	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5706	*
5707	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5708	*/
5709	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5710	{
5711	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5712
5713	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5714	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
5715	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
5716
5717	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5718	#ifdef DEBUG_bird
5719	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5720	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5721	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5722	#endif
5723	}
5724	return VINF_SUCCESS;
5725	}
5726
5727
5728	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5729	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5730	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5731	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5732	#else
5733	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5734	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5735	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5736	#endif
5737
5738
5739	/**
5740	* Saves the guest segment registers from the current VMCS into the guest-CPU
5741	* context.
5742	*
5743	* @returns VBox status code.
5744	* @param pVCpu Pointer to the VMCPU.
5745	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5746	* out-of-sync. Make sure to update the required fields
5747	* before using them.
5748	*
5749	* @remarks No-long-jump zone!!!
5750	*/
5751	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5752	{
5753	/* Guest segment registers. */
5754	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5755	{
5756	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5757	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5758	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5759	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5760	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5761	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5762	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5763
5764	/* Restore segment attributes for real-on-v86 mode hack. */
5765	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5766	{
5767	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5768	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5769	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5770	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5771	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5772	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5773	}
5774	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5775	}
5776
5777	return VINF_SUCCESS;
5778	}
5779
5780
5781	/**
5782	* Saves the guest descriptor table registers and task register from the current
5783	* VMCS into the guest-CPU context.
5784	*
5785	* @returns VBox status code.
5786	* @param pVCpu Pointer to the VMCPU.
5787	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5788	* out-of-sync. Make sure to update the required fields
5789	* before using them.
5790	*
5791	* @remarks No-long-jump zone!!!
5792	*/
5793	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5794	{
5795	int rc = VINF_SUCCESS;
5796
5797	/* Guest LDTR. */
5798	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5799	{
5800	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5801	AssertRCReturn(rc, rc);
5802	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5803	}
5804
5805	/* Guest GDTR. */
5806	uint64_t u64Val = 0;
5807	uint32_t u32Val = 0;
5808	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5809	{
5810	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5811	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5812	pMixedCtx->gdtr.pGdt = u64Val;
5813	pMixedCtx->gdtr.cbGdt = u32Val;
5814	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5815	}
5816
5817	/* Guest IDTR. */
5818	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5819	{
5820	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5821	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5822	pMixedCtx->idtr.pIdt = u64Val;
5823	pMixedCtx->idtr.cbIdt = u32Val;
5824	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5825	}
5826
5827	/* Guest TR. */
5828	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5829	{
5830	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5831	AssertRCReturn(rc, rc);
5832
5833	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5834	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5835	{
5836	rc = VMXLOCAL_READ_SEG(TR, tr);
5837	AssertRCReturn(rc, rc);
5838	}
5839	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5840	}
5841	return rc;
5842	}
5843
5844	#undef VMXLOCAL_READ_SEG
5845
5846
5847	/**
5848	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5849	* context.
5850	*
5851	* @returns VBox status code.
5852	* @param pVCpu Pointer to the VMCPU.
5853	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5854	* out-of-sync. Make sure to update the required fields
5855	* before using them.
5856	*
5857	* @remarks No-long-jump zone!!!
5858	*/
5859	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5860	{
5861	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5862	{
5863	if (!pVCpu->hm.s.fUsingHyperDR7)
5864	{
5865	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5866	uint32_t u32Val;
5867	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5868	pMixedCtx->dr[7] = u32Val;
5869	}
5870
5871	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5872	}
5873	return VINF_SUCCESS;
5874	}
5875
5876
5877	/**
5878	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
5879	*
5880	* @returns VBox status code.
5881	* @param pVCpu Pointer to the VMCPU.
5882	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5883	* out-of-sync. Make sure to update the required fields
5884	* before using them.
5885	*
5886	* @remarks No-long-jump zone!!!
5887	*/
5888	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5889	{
5890	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5891	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5892	return VINF_SUCCESS;
5893	}
5894
5895
5896	/**
5897	* Saves the entire guest state from the currently active VMCS into the
5898	* guest-CPU context. This essentially VMREADs all guest-data.
5899	*
5900	* @returns VBox status code.
5901	* @param pVCpu Pointer to the VMCPU.
5902	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5903	* out-of-sync. Make sure to update the required fields
5904	* before using them.
5905	*/
5906	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5907	{
5908	Assert(pVCpu);
5909	Assert(pMixedCtx);
5910
5911	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5912	return VINF_SUCCESS;
5913
5914	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
5915	again on the ring-3 callback path, there is no real need to. */
5916	if (VMMRZCallRing3IsEnabled(pVCpu))
5917	VMMR0LogFlushDisable(pVCpu);
5918	else
5919	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5920	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5921
5922	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5923	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5924
5925	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5926	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5927
5928	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5929	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5930
5931	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5932	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5933
5934	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5935	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5936
5937	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5938	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5939
5940	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5941	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5942
5943	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5944	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5945
5946	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5947	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5948
5949	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5950	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5951
5952	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5953	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5954
5955	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5956	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5957
5958	if (VMMRZCallRing3IsEnabled(pVCpu))
5959	VMMR0LogFlushEnable(pVCpu);
5960
5961	return rc;
5962	}
5963
5964
5965	/**
5966	* Check per-VM and per-VCPU force flag actions that require us to go back to
5967	* ring-3 for one reason or another.
5968	*
5969	* @returns VBox status code (information status code included).
5970	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5971	* ring-3.
5972	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5973	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5974	* interrupts)
5975	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5976	* all EMTs to be in ring-3.
5977	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5978	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5979	* to the EM loop.
5980	*
5981	* @param pVM Pointer to the VM.
5982	* @param pVCpu Pointer to the VMCPU.
5983	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5984	* out-of-sync. Make sure to update the required fields
5985	* before using them.
5986	*/
5987	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5988	{
5989	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5990
5991	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5992	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5993	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5994	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5995	{
5996	/* We need the control registers now, make sure the guest-CPU context is updated. */
5997	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5998	AssertRCReturn(rc3, rc3);
5999
6000	/* Pending HM CR3 sync. */
6001	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6002	{
6003	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6004	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6005	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6006	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6007	}
6008
6009	/* Pending HM PAE PDPEs. */
6010	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6011	{
6012	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6013	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6014	}
6015
6016	/* Pending PGM C3 sync. */
6017	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6018	{
6019	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6020	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6021	if (rc2 != VINF_SUCCESS)
6022	{
6023	AssertRC(rc2);
6024	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6025	return rc2;
6026	}
6027	}
6028
6029	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6030	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6031	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6032	{
6033	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6034	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6035	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6036	return rc2;
6037	}
6038
6039	/* Pending VM request packets, such as hardware interrupts. */
6040	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6041	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6042	{
6043	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6044	return VINF_EM_PENDING_REQUEST;
6045	}
6046
6047	/* Pending PGM pool flushes. */
6048	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6049	{
6050	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6051	return VINF_PGM_POOL_FLUSH_PENDING;
6052	}
6053
6054	/* Pending DMA requests. */
6055	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6056	{
6057	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6058	return VINF_EM_RAW_TO_R3;
6059	}
6060	}
6061
6062	return VINF_SUCCESS;
6063	}
6064
6065
6066	/**
6067	* Converts any TRPM trap into a pending HM event. This is typically used when
6068	* entering from ring-3 (not longjmp returns).
6069	*
6070	* @param pVCpu Pointer to the VMCPU.
6071	*/
6072	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6073	{
6074	Assert(TRPMHasTrap(pVCpu));
6075	Assert(!pVCpu->hm.s.Event.fPending);
6076
6077	uint8_t uVector;
6078	TRPMEVENT enmTrpmEvent;
6079	RTGCUINT uErrCode;
6080	RTGCUINTPTR GCPtrFaultAddress;
6081	uint8_t cbInstr;
6082
6083	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6084	AssertRC(rc);
6085
6086	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6087	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6088	if (enmTrpmEvent == TRPM_TRAP)
6089	{
6090	switch (uVector)
6091	{
6092	case X86_XCPT_BP:
6093	case X86_XCPT_OF:
6094	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6095	break;
6096
6097	case X86_XCPT_PF:
6098	case X86_XCPT_DF:
6099	case X86_XCPT_TS:
6100	case X86_XCPT_NP:
6101	case X86_XCPT_SS:
6102	case X86_XCPT_GP:
6103	case X86_XCPT_AC:
6104	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6105	/* no break! */
6106	default:
6107	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6108	break;
6109	}
6110	}
6111	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6112	{
6113	if (uVector == X86_XCPT_NMI)
6114	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6115	else
6116	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6117	}
6118	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6119	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6120	else
6121	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6122
6123	rc = TRPMResetTrap(pVCpu);
6124	AssertRC(rc);
6125	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6126	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6127
6128	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6129	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6130	}
6131
6132
6133	/**
6134	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6135	* VT-x to execute any instruction.
6136	*
6137	* @param pvCpu Pointer to the VMCPU.
6138	*/
6139	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6140	{
6141	Assert(pVCpu->hm.s.Event.fPending);
6142
6143	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6144	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6145	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6146	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6147
6148	/* If a trap was already pending, we did something wrong! */
6149	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6150
6151	TRPMEVENT enmTrapType;
6152	switch (uVectorType)
6153	{
6154	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6155	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6156	enmTrapType = TRPM_HARDWARE_INT;
6157	break;
6158
6159	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6160	enmTrapType = TRPM_SOFTWARE_INT;
6161	break;
6162
6163	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6164	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6165	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6166	enmTrapType = TRPM_TRAP;
6167	break;
6168
6169	default:
6170	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6171	enmTrapType = TRPM_32BIT_HACK;
6172	break;
6173	}
6174
6175	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6176
6177	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6178	AssertRC(rc);
6179
6180	if (fErrorCodeValid)
6181	TRPMSetErrorCode(pVCpu, uErrorCode);
6182
6183	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6184	&& uVector == X86_XCPT_PF)
6185	{
6186	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6187	}
6188	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6189	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6190	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6191	{
6192	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6193	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6194	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6195	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6196	}
6197	pVCpu->hm.s.Event.fPending = false;
6198	}
6199
6200
6201	/**
6202	* Does the necessary state syncing before returning to ring-3 for any reason
6203	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6204	*
6205	* @returns VBox status code.
6206	* @param pVM Pointer to the VM.
6207	* @param pVCpu Pointer to the VMCPU.
6208	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6209	* be out-of-sync. Make sure to update the required
6210	* fields before using them.
6211	* @param fSaveGuestState Whether to save the guest state or not.
6212	*
6213	* @remarks If you modify code here, make sure to check whether
6214	* hmR0VmxCallRing3Callback() needs to be updated too.
6215	* @remarks No-long-jmp zone!!!
6216	*/
6217	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6218	{
6219	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6220	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6221
6222	RTCPUID idCpu = RTMpCpuId();
6223	Log4Func(("HostCpuId=%u\n", idCpu));
6224
6225	/* Save the guest state if necessary. */
6226	if ( fSaveGuestState
6227	&& pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6228	{
6229	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6230	AssertRCReturn(rc, rc);
6231	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6232	}
6233
6234	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6235	if (CPUMIsGuestFPUStateActive(pVCpu))
6236	{
6237	/* We shouldn't reload CR0 without saving it first. */
6238	if (!fSaveGuestState)
6239	{
6240	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6241	AssertRCReturn(rc, rc);
6242	}
6243	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6244	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6245	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6246	}
6247
6248	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6249	#ifdef VBOX_STRICT
6250	if (CPUMIsHyperDebugStateActive(pVCpu))
6251	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6252	#endif
6253	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6254	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6255	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6256	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6257
6258	#if HC_ARCH_BITS == 64
6259	/* Restore host-state bits that VT-x only restores partially. */
6260	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6261	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6262	{
6263	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6264	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6265	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6266	}
6267	#endif
6268
6269	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6270	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6271	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6272	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6273	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6274	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6275	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6276	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6277
6278	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6279
6280	/** @todo This kinda defeats the purpose of having preemption hooks.
6281	* The problem is, deregistering the hooks should be moved to a place that
6282	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6283	* context.
6284	*/
6285	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6286	{
6287	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6288	AssertRCReturn(rc, rc);
6289
6290	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6291	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6292	}
6293	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6294	NOREF(idCpu);
6295
6296	return VINF_SUCCESS;
6297	}
6298
6299
6300	/**
6301	* Leaves the VT-x session.
6302	*
6303	* @returns VBox status code.
6304	* @param pVM Pointer to the VM.
6305	* @param pVCpu Pointer to the VMCPU.
6306	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6307	* out-of-sync. Make sure to update the required fields
6308	* before using them.
6309	*
6310	* @remarks No-long-jmp zone!!!
6311	*/
6312	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6313	{
6314	HM_DISABLE_PREEMPT_IF_NEEDED();
6315	HMVMX_ASSERT_CPU_SAFE();
6316	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6317	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6318
6319	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6320	and done this from the VMXR0ThreadCtxCallback(). */
6321	if (!pVCpu->hm.s.fLeaveDone)
6322	{
6323	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6324	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6325	pVCpu->hm.s.fLeaveDone = true;
6326	}
6327
6328	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6329	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6330	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6331	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6332	VMMR0ThreadCtxHooksDeregister(pVCpu);
6333
6334	/* Leave HM context. This takes care of local init (term). */
6335	int rc = HMR0LeaveCpu(pVCpu);
6336
6337	HM_RESTORE_PREEMPT_IF_NEEDED();
6338
6339	return rc;
6340	}
6341
6342
6343	/**
6344	* Does the necessary state syncing before doing a longjmp to ring-3.
6345	*
6346	* @returns VBox status code.
6347	* @param pVM Pointer to the VM.
6348	* @param pVCpu Pointer to the VMCPU.
6349	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6350	* out-of-sync. Make sure to update the required fields
6351	* before using them.
6352	*
6353	* @remarks No-long-jmp zone!!!
6354	*/
6355	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6356	{
6357	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6358	}
6359
6360
6361	/**
6362	* Take necessary actions before going back to ring-3.
6363	*
6364	* An action requires us to go back to ring-3. This function does the necessary
6365	* steps before we can safely return to ring-3. This is not the same as longjmps
6366	* to ring-3, this is voluntary and prepares the guest so it may continue
6367	* executing outside HM (recompiler/IEM).
6368	*
6369	* @returns VBox status code.
6370	* @param pVM Pointer to the VM.
6371	* @param pVCpu Pointer to the VMCPU.
6372	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6373	* out-of-sync. Make sure to update the required fields
6374	* before using them.
6375	* @param rcExit The reason for exiting to ring-3. Can be
6376	* VINF_VMM_UNKNOWN_RING3_CALL.
6377	*/
6378	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6379	{
6380	Assert(pVM);
6381	Assert(pVCpu);
6382	Assert(pMixedCtx);
6383	HMVMX_ASSERT_PREEMPT_SAFE();
6384
6385	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6386	{
6387	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6388	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6389	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6390	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6391	}
6392
6393	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6394	VMMRZCallRing3Disable(pVCpu);
6395	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6396
6397	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6398	if (pVCpu->hm.s.Event.fPending)
6399	{
6400	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6401	Assert(!pVCpu->hm.s.Event.fPending);
6402	}
6403
6404	/* Save guest state and restore host state bits. */
6405	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6406	AssertRCReturn(rc, rc);
6407	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6408
6409	/* Sync recompiler state. */
6410	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6411	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6412	\| CPUM_CHANGED_LDTR
6413	\| CPUM_CHANGED_GDTR
6414	\| CPUM_CHANGED_IDTR
6415	\| CPUM_CHANGED_TR
6416	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6417	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6418	if ( pVM->hm.s.fNestedPaging
6419	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6420	{
6421	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6422	}
6423
6424	Assert(!pVCpu->hm.s.fClearTrapFlag);
6425
6426	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6427	if (rcExit != VINF_EM_RAW_INTERRUPT)
6428	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6429
6430	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6431
6432	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6433	VMMRZCallRing3RemoveNotification(pVCpu);
6434	VMMRZCallRing3Enable(pVCpu);
6435
6436	return rc;
6437	}
6438
6439
6440	/**
6441	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6442	* longjump to ring-3 and possibly get preempted.
6443	*
6444	* @returns VBox status code.
6445	* @param pVCpu Pointer to the VMCPU.
6446	* @param enmOperation The operation causing the ring-3 longjump.
6447	* @param pvUser Opaque pointer to the guest-CPU context. The data
6448	* may be out-of-sync. Make sure to update the required
6449	* fields before using them.
6450	* @remarks If you modify code here, make sure to check whether
6451	* hmR0VmxLeave() needs to be updated too.
6452	*/
6453	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6454	{
6455	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6456	{
6457	VMMRZCallRing3RemoveNotification(pVCpu);
6458	HM_DISABLE_PREEMPT_IF_NEEDED();
6459
6460	/* If anything here asserts or fails, good luck. */
6461	if (CPUMIsGuestFPUStateActive(pVCpu))
6462	CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6463
6464	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6465
6466	#if HC_ARCH_BITS == 64
6467	/* Restore host-state bits that VT-x only restores partially. */
6468	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6469	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6470	{
6471	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6472	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6473	}
6474	#endif
6475	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6476	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6477	{
6478	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6479	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6480	}
6481
6482	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6483	VMMR0ThreadCtxHooksDeregister(pVCpu);
6484
6485	HMR0LeaveCpu(pVCpu);
6486	HM_RESTORE_PREEMPT_IF_NEEDED();
6487	return VINF_SUCCESS;
6488	}
6489
6490	Assert(pVCpu);
6491	Assert(pvUser);
6492	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6493	HMVMX_ASSERT_PREEMPT_SAFE();
6494
6495	VMMRZCallRing3Disable(pVCpu);
6496	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6497
6498	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6499	enmOperation));
6500
6501	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6502	AssertRCReturn(rc, rc);
6503
6504	VMMRZCallRing3Enable(pVCpu);
6505	return VINF_SUCCESS;
6506	}
6507
6508
6509	/**
6510	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6511	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6512	*
6513	* @param pVCpu Pointer to the VMCPU.
6514	*/
6515	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6516	{
6517	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6518	{
6519	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6520	{
6521	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6522	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6523	AssertRC(rc);
6524	}
6525	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6526	}
6527
6528
6529	/**
6530	* Evaluates the event to be delivered to the guest and sets it as the pending
6531	* event.
6532	*
6533	* @param pVCpu Pointer to the VMCPU.
6534	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6535	* out-of-sync. Make sure to update the required fields
6536	* before using them.
6537	*/
6538	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6539	{
6540	Assert(!pVCpu->hm.s.Event.fPending);
6541
6542	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6543	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6544	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6545	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6546
6547	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6548	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6549	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6550	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6551	Assert(!TRPMHasTrap(pVCpu));
6552
6553	/** @todo SMI. SMIs take priority over NMIs. */
6554	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6555	{
6556	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6557	if ( !fBlockMovSS
6558	&& !fBlockSti)
6559	{
6560	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6561	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6562	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6563	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6564
6565	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6566	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6567	}
6568	else
6569	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6570	}
6571	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6572	&& !pVCpu->hm.s.fSingleInstruction)
6573	{
6574	/*
6575	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6576	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6577	* evaluated here and not set as pending, solely based on the force-flags.
6578	*/
6579	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6580	AssertRC(rc);
6581	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6582	if ( !fBlockInt
6583	&& !fBlockSti
6584	&& !fBlockMovSS)
6585	{
6586	uint8_t u8Interrupt;
6587	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6588	if (RT_SUCCESS(rc))
6589	{
6590	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6591	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6592	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6593
6594	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
6595	}
6596	else
6597	{
6598	/** @todo Does this actually happen? If not turn it into an assertion. */
6599	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6600	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6601	}
6602	}
6603	else
6604	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6605	}
6606	}
6607
6608
6609	/**
6610	* Injects any pending events into the guest if the guest is in a state to
6611	* receive them.
6612	*
6613	* @returns VBox status code (informational status codes included).
6614	* @param pVCpu Pointer to the VMCPU.
6615	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6616	* out-of-sync. Make sure to update the required fields
6617	* before using them.
6618	*/
6619	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6620	{
6621	HMVMX_ASSERT_PREEMPT_SAFE();
6622	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6623
6624	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6625	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6626	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6627	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6628
6629	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6630	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6631	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6632	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6633	Assert(!TRPMHasTrap(pVCpu));
6634
6635	int rc = VINF_SUCCESS;
6636	if (pVCpu->hm.s.Event.fPending)
6637	{
6638	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
6639	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6640	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6641	{
6642	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6643	AssertRCReturn(rc, rc);
6644	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6645	Assert(!fBlockInt);
6646	Assert(!fBlockSti);
6647	Assert(!fBlockMovSS);
6648	}
6649	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6650	{
6651	Assert(!fBlockSti);
6652	Assert(!fBlockMovSS);
6653	}
6654	#endif
6655	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
6656	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
6657	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6658	AssertRCReturn(rc, rc);
6659
6660	/* Update the interruptibility-state as it could have been changed by
6661	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6662	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6663	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6664
6665	#ifdef VBOX_WITH_STATISTICS
6666	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6667	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6668	else
6669	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6670	#endif
6671	}
6672
6673	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6674	int rc2 = VINF_SUCCESS;
6675	if ( fBlockSti
6676	\|\| fBlockMovSS)
6677	{
6678	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6679	{
6680	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6681	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6682	{
6683	/*
6684	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6685	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6686	* See Intel spec. 27.3.4 "Saving Non-Register State".
6687	*/
6688	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6689	AssertRCReturn(rc, rc);
6690	}
6691	}
6692	else
6693	{
6694	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6695	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6696	uIntrState = 0;
6697	}
6698	}
6699
6700	/*
6701	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6702	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6703	*/
6704	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6705	AssertRC(rc2);
6706
6707	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6708	return rc;
6709	}
6710
6711
6712	/**
6713	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6714	*
6715	* @param pVCpu Pointer to the VMCPU.
6716	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6717	* out-of-sync. Make sure to update the required fields
6718	* before using them.
6719	*/
6720	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6721	{
6722	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6723	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6724	}
6725
6726
6727	/**
6728	* Injects a double-fault (#DF) exception into the VM.
6729	*
6730	* @returns VBox status code (informational status code included).
6731	* @param pVCpu Pointer to the VMCPU.
6732	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6733	* out-of-sync. Make sure to update the required fields
6734	* before using them.
6735	*/
6736	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6737	{
6738	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6739	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6740	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6741	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6742	puIntrState);
6743	}
6744
6745
6746	/**
6747	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6748	*
6749	* @param pVCpu Pointer to the VMCPU.
6750	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6751	* out-of-sync. Make sure to update the required fields
6752	* before using them.
6753	*/
6754	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6755	{
6756	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6757	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6758	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6759	}
6760
6761
6762	/**
6763	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6764	*
6765	* @param pVCpu Pointer to the VMCPU.
6766	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6767	* out-of-sync. Make sure to update the required fields
6768	* before using them.
6769	* @param cbInstr The value of RIP that is to be pushed on the guest
6770	* stack.
6771	*/
6772	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6773	{
6774	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6775	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6776	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6777	}
6778
6779
6780	/**
6781	* Injects a general-protection (#GP) fault into the VM.
6782	*
6783	* @returns VBox status code (informational status code included).
6784	* @param pVCpu Pointer to the VMCPU.
6785	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6786	* out-of-sync. Make sure to update the required fields
6787	* before using them.
6788	* @param u32ErrorCode The error code associated with the #GP.
6789	*/
6790	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6791	uint32_t *puIntrState)
6792	{
6793	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6794	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6795	if (fErrorCodeValid)
6796	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6797	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6798	puIntrState);
6799	}
6800
6801
6802	/**
6803	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6804	*
6805	* @param pVCpu Pointer to the VMCPU.
6806	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6807	* out-of-sync. Make sure to update the required fields
6808	* before using them.
6809	* @param uVector The software interrupt vector number.
6810	* @param cbInstr The value of RIP that is to be pushed on the guest
6811	* stack.
6812	*/
6813	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6814	{
6815	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6816	if ( uVector == X86_XCPT_BP
6817	\|\| uVector == X86_XCPT_OF)
6818	{
6819	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6820	}
6821	else
6822	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6823	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6824	}
6825
6826
6827	/**
6828	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6829	* stack.
6830	*
6831	* @returns VBox status code (information status code included).
6832	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6833	* @param pVM Pointer to the VM.
6834	* @param pMixedCtx Pointer to the guest-CPU context.
6835	* @param uValue The value to push to the guest stack.
6836	*/
6837	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6838	{
6839	/*
6840	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6841	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6842	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6843	*/
6844	if (pMixedCtx->sp == 1)
6845	return VINF_EM_RESET;
6846	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6847	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6848	AssertRCReturn(rc, rc);
6849	return rc;
6850	}
6851
6852
6853	/**
6854	* Injects an event into the guest upon VM-entry by updating the relevant fields
6855	* in the VM-entry area in the VMCS.
6856	*
6857	* @returns VBox status code (informational error codes included).
6858	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6859	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6860	*
6861	* @param pVCpu Pointer to the VMCPU.
6862	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6863	* be out-of-sync. Make sure to update the required
6864	* fields before using them.
6865	* @param u64IntInfo The VM-entry interruption-information field.
6866	* @param cbInstr The VM-entry instruction length in bytes (for
6867	* software interrupts, exceptions and privileged
6868	* software exceptions).
6869	* @param u32ErrCode The VM-entry exception error code.
6870	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6871	* @param puIntrState Pointer to the current guest interruptibility-state.
6872	* This interruptibility-state will be updated if
6873	* necessary. This cannot not be NULL.
6874	*
6875	* @remarks Requires CR0!
6876	* @remarks No-long-jump zone!!!
6877	*/
6878	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
6879	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6880	{
6881	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6882	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
6883	Assert(puIntrState);
6884	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
6885
6886	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
6887	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
6888
6889	#ifdef VBOX_STRICT
6890	/* Validate the error-code-valid bit for hardware exceptions. */
6891	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6892	{
6893	switch (uVector)
6894	{
6895	case X86_XCPT_PF:
6896	case X86_XCPT_DF:
6897	case X86_XCPT_TS:
6898	case X86_XCPT_NP:
6899	case X86_XCPT_SS:
6900	case X86_XCPT_GP:
6901	case X86_XCPT_AC:
6902	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
6903	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6904	/* fallthru */
6905	default:
6906	break;
6907	}
6908	}
6909	#endif
6910
6911	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6912	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6913	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6914
6915	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6916
6917	/* We require CR0 to check if the guest is in real-mode. */
6918	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6919	AssertRCReturn(rc, rc);
6920
6921	/*
6922	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6923	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6924	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6925	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6926	*/
6927	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6928	{
6929	PVM pVM = pVCpu->CTX_SUFF(pVM);
6930	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6931	{
6932	Assert(PDMVmmDevHeapIsEnabled(pVM));
6933	Assert(pVM->hm.s.vmx.pRealModeTSS);
6934
6935	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6936	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6937	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6938	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6939	AssertRCReturn(rc, rc);
6940	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6941
6942	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6943	const size_t cbIdtEntry = sizeof(X86IDTR16);
6944	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6945	{
6946	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6947	if (uVector == X86_XCPT_DF)
6948	return VINF_EM_RESET;
6949	else if (uVector == X86_XCPT_GP)
6950	{
6951	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6952	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6953	}
6954
6955	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6956	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6957	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6958	}
6959
6960	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6961	uint16_t uGuestIp = pMixedCtx->ip;
6962	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6963	{
6964	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6965	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6966	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6967	}
6968	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6969	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6970
6971	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6972	X86IDTR16 IdtEntry;
6973	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6974	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6975	AssertRCReturn(rc, rc);
6976
6977	/* Construct the stack frame for the interrupt/exception handler. */
6978	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6979	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6980	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6981	AssertRCReturn(rc, rc);
6982
6983	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6984	if (rc == VINF_SUCCESS)
6985	{
6986	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6987	pMixedCtx->rip = IdtEntry.offSel;
6988	pMixedCtx->cs.Sel = IdtEntry.uSel;
6989	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
6990	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6991	&& uVector == X86_XCPT_PF)
6992	{
6993	pMixedCtx->cr2 = GCPtrFaultAddress;
6994	}
6995
6996	/* If any other guest-state bits are changed here, make sure to update
6997	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6998	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
6999	\| HM_CHANGED_GUEST_RIP
7000	\| HM_CHANGED_GUEST_RFLAGS
7001	\| HM_CHANGED_GUEST_RSP);
7002
7003	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7004	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7005	{
7006	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7007	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7008	Log4(("Clearing inhibition due to STI.\n"));
7009	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7010	}
7011	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7012
7013	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7014	it, if we are returning to ring-3 before executing guest code. */
7015	pVCpu->hm.s.Event.fPending = false;
7016	}
7017	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7018	return rc;
7019	}
7020	else
7021	{
7022	/*
7023	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7024	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7025	*/
7026	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7027	}
7028	}
7029
7030	/* Validate. */
7031	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7032	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7033	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7034
7035	/* Inject. */
7036	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7037	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7038	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7039	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7040
7041	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7042	&& uVector == X86_XCPT_PF)
7043	{
7044	pMixedCtx->cr2 = GCPtrFaultAddress;
7045	}
7046
7047	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7048	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7049
7050	AssertRCReturn(rc, rc);
7051	return rc;
7052	}
7053
7054
7055	/**
7056	* Clears the interrupt-window exiting control in the VMCS and if necessary
7057	* clears the current event in the VMCS as well.
7058	*
7059	* @returns VBox status code.
7060	* @param pVCpu Pointer to the VMCPU.
7061	*
7062	* @remarks Use this function only to clear events that have not yet been
7063	* delivered to the guest but are injected in the VMCS!
7064	* @remarks No-long-jump zone!!!
7065	*/
7066	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7067	{
7068	int rc;
7069	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7070
7071	/* Clear interrupt-window exiting control. */
7072	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7073	{
7074	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7075	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7076	AssertRC(rc);
7077	}
7078
7079	if (!pVCpu->hm.s.Event.fPending)
7080	return;
7081
7082	#ifdef VBOX_STRICT
7083	uint32_t u32EntryInfo;
7084	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7085	AssertRC(rc);
7086	Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
7087	#endif
7088
7089	/* Clear the entry-interruption field (including the valid bit). */
7090	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7091	AssertRC(rc);
7092
7093	/* Clear the pending debug exception field. */
7094	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7095	AssertRC(rc);
7096	}
7097
7098
7099	/**
7100	* Enters the VT-x session.
7101	*
7102	* @returns VBox status code.
7103	* @param pVM Pointer to the VM.
7104	* @param pVCpu Pointer to the VMCPU.
7105	* @param pCpu Pointer to the CPU info struct.
7106	*/
7107	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7108	{
7109	AssertPtr(pVM);
7110	AssertPtr(pVCpu);
7111	Assert(pVM->hm.s.vmx.fSupported);
7112	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7113	NOREF(pCpu);
7114
7115	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7116	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7117
7118	#ifdef VBOX_STRICT
7119	/* Make sure we're in VMX root mode. */
7120	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7121	if (!(u32HostCR4 & X86_CR4_VMXE))
7122	{
7123	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7124	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7125	}
7126	#endif
7127
7128	/*
7129	* Load the VCPU's VMCS as the current (and active) one.
7130	*/
7131	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7132	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7133	if (RT_FAILURE(rc))
7134	return rc;
7135
7136	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7137	pVCpu->hm.s.fLeaveDone = false;
7138	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7139
7140	return VINF_SUCCESS;
7141	}
7142
7143
7144	/**
7145	* The thread-context callback (only on platforms which support it).
7146	*
7147	* @param enmEvent The thread-context event.
7148	* @param pVCpu Pointer to the VMCPU.
7149	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7150	* @thread EMT(pVCpu)
7151	*/
7152	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7153	{
7154	switch (enmEvent)
7155	{
7156	case RTTHREADCTXEVENT_PREEMPTING:
7157	{
7158	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7159	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7160	VMCPU_ASSERT_EMT(pVCpu);
7161
7162	PVM pVM = pVCpu->CTX_SUFF(pVM);
7163	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7164
7165	/* No longjmps (logger flushes, locks) in this fragile context. */
7166	VMMRZCallRing3Disable(pVCpu);
7167	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7168
7169	/*
7170	* Restore host-state (FPU, debug etc.)
7171	*/
7172	if (!pVCpu->hm.s.fLeaveDone)
7173	{
7174	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7175	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7176	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7177	pVCpu->hm.s.fLeaveDone = true;
7178	}
7179
7180	/* Leave HM context, takes care of local init (term). */
7181	int rc = HMR0LeaveCpu(pVCpu);
7182	AssertRC(rc); NOREF(rc);
7183
7184	/* Restore longjmp state. */
7185	VMMRZCallRing3Enable(pVCpu);
7186	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7187	break;
7188	}
7189
7190	case RTTHREADCTXEVENT_RESUMED:
7191	{
7192	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7193	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7194	VMCPU_ASSERT_EMT(pVCpu);
7195
7196	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7197	VMMRZCallRing3Disable(pVCpu);
7198	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7199
7200	/* Initialize the bare minimum state required for HM. This takes care of
7201	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7202	int rc = HMR0EnterCpu(pVCpu);
7203	AssertRC(rc);
7204	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7205
7206	/* Load the active VMCS as the current one. */
7207	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7208	{
7209	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7210	AssertRC(rc); NOREF(rc);
7211	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7212	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7213	}
7214	pVCpu->hm.s.fLeaveDone = false;
7215
7216	/* Restore longjmp state. */
7217	VMMRZCallRing3Enable(pVCpu);
7218	break;
7219	}
7220
7221	default:
7222	break;
7223	}
7224	}
7225
7226
7227	/**
7228	* Saves the host state in the VMCS host-state.
7229	* Sets up the VM-exit MSR-load area.
7230	*
7231	* The CPU state will be loaded from these fields on every successful VM-exit.
7232	*
7233	* @returns VBox status code.
7234	* @param pVM Pointer to the VM.
7235	* @param pVCpu Pointer to the VMCPU.
7236	*
7237	* @remarks No-long-jump zone!!!
7238	*/
7239	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7240	{
7241	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7242
7243	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7244	return VINF_SUCCESS;
7245
7246	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7247	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7248
7249	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7250	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7251
7252	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7253	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7254
7255	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7256	return rc;
7257	}
7258
7259
7260	/**
7261	* Saves the host state in the VMCS host-state.
7262	*
7263	* @returns VBox status code.
7264	* @param pVM Pointer to the VM.
7265	* @param pVCpu Pointer to the VMCPU.
7266	*
7267	* @remarks No-long-jump zone!!!
7268	*/
7269	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7270	{
7271	AssertPtr(pVM);
7272	AssertPtr(pVCpu);
7273
7274	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7275
7276	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7277	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7278	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7279	return hmR0VmxSaveHostState(pVM, pVCpu);
7280	}
7281
7282
7283	/**
7284	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7285	* loaded from these fields on every successful VM-entry.
7286	*
7287	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7288	* Sets up the VM-entry controls.
7289	* Sets up the appropriate VMX non-root function to execute guest code based on
7290	* the guest CPU mode.
7291	*
7292	* @returns VBox status code.
7293	* @param pVM Pointer to the VM.
7294	* @param pVCpu Pointer to the VMCPU.
7295	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7296	* out-of-sync. Make sure to update the required fields
7297	* before using them.
7298	*
7299	* @remarks No-long-jump zone!!!
7300	*/
7301	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7302	{
7303	AssertPtr(pVM);
7304	AssertPtr(pVCpu);
7305	AssertPtr(pMixedCtx);
7306	HMVMX_ASSERT_PREEMPT_SAFE();
7307
7308	#ifdef LOG_ENABLED
7309	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7310	* probably not initialized yet? Anyway this will do for now.
7311	*
7312	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7313	* interface and disable ring-3 calls when thread-context hooks are not
7314	* available. */
7315	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7316	VMMR0LogFlushDisable(pVCpu);
7317	#endif
7318
7319	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7320
7321	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7322
7323	/* Determine real-on-v86 mode. */
7324	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7325	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7326	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7327	{
7328	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7329	}
7330
7331	/*
7332	* Load the guest-state into the VMCS.
7333	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7334	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7335	*/
7336	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7337	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7338
7339	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7340	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7341	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7342
7343	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7344	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7345	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7346
7347	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7348	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7349
7350	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7351	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7352
7353	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7354	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7355	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7356
7357	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7358	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7359
7360	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7361	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7362
7363	/*
7364	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7365	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7366	*/
7367	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7368	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7369
7370	/* Clear any unused and reserved bits. */
7371	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7372
7373	#ifdef LOG_ENABLED
7374	/* Only reenable log-flushing if the caller has it enabled. */
7375	if (!fCallerDisabledLogFlush)
7376	VMMR0LogFlushEnable(pVCpu);
7377	#endif
7378
7379	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7380	return rc;
7381	}
7382
7383
7384	/**
7385	* Loads the state shared between the host and guest into the VMCS.
7386	*
7387	* @param pVM Pointer to the VM.
7388	* @param pVCpu Pointer to the VMCPU.
7389	* @param pCtx Pointer to the guest-CPU context.
7390	*
7391	* @remarks No-long-jump zone!!!
7392	*/
7393	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7394	{
7395	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7396	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7397
7398	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7399	{
7400	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7401	AssertRC(rc);
7402	}
7403
7404	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7405	{
7406	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7407	AssertRC(rc);
7408
7409	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7410	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7411	{
7412	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7413	AssertRC(rc);
7414	}
7415	}
7416
7417	AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7418	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7419	}
7420
7421
7422	/**
7423	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7424	*
7425	* @param pVM Pointer to the VM.
7426	* @param pVCpu Pointer to the VMCPU.
7427	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7428	* out-of-sync. Make sure to update the required fields
7429	* before using them.
7430	*/
7431	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7432	{
7433	HMVMX_ASSERT_PREEMPT_SAFE();
7434
7435	Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7436	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7437	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7438	#endif
7439
7440	if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7441	{
7442	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7443	AssertRC(rc);
7444	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7445	}
7446	else if (VMCPU_HMCF_VALUE(pVCpu))
7447	{
7448	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7449	AssertRC(rc);
7450	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7451	}
7452
7453	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7454	AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7455	\|\| VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7456	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7457
7458	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7459	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7460	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7461	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7462	#endif
7463	}
7464
7465
7466	/**
7467	* Does the preparations before executing guest code in VT-x.
7468	*
7469	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7470	* recompiler. We must be cautious what we do here regarding committing
7471	* guest-state information into the VMCS assuming we assuredly execute the
7472	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7473	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7474	* so that the recompiler can (and should) use them when it resumes guest
7475	* execution. Otherwise such operations must be done when we can no longer
7476	* exit to ring-3.
7477	*
7478	* @returns Strict VBox status code.
7479	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7480	* have been disabled.
7481	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7482	* double-fault into the guest.
7483	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7484	*
7485	* @param pVM Pointer to the VM.
7486	* @param pVCpu Pointer to the VMCPU.
7487	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7488	* out-of-sync. Make sure to update the required fields
7489	* before using them.
7490	* @param pVmxTransient Pointer to the VMX transient structure.
7491	*
7492	* @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7493	* interrupts will be disabled.
7494	*/
7495	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7496	{
7497	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7498
7499	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7500	PGMRZDynMapFlushAutoSet(pVCpu);
7501	#endif
7502
7503	/* Check force flag actions that might require us to go back to ring-3. */
7504	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7505	if (rc != VINF_SUCCESS)
7506	return rc;
7507
7508	#ifndef IEM_VERIFICATION_MODE_FULL
7509	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7510	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7511	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7512	{
7513	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7514	RTGCPHYS GCPhysApicBase;
7515	GCPhysApicBase = pMixedCtx->msrApicBase;
7516	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7517
7518	/* Unalias any existing mapping. */
7519	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7520	AssertRCReturn(rc, rc);
7521
7522	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7523	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7524	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7525	AssertRCReturn(rc, rc);
7526
7527	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7528	}
7529	#endif /* !IEM_VERIFICATION_MODE_FULL */
7530
7531	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7532	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7533
7534	/*
7535	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7536	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7537	*/
7538	if (TRPMHasTrap(pVCpu))
7539	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7540	else if (!pVCpu->hm.s.Event.fPending)
7541	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7542
7543	/*
7544	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7545	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7546	*/
7547	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7548	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7549	{
7550	Assert(rc == VINF_EM_RESET);
7551	return rc;
7552	}
7553
7554	/*
7555	* No longjmps to ring-3 from this point on!!!
7556	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7557	* This also disables flushing of the R0-logger instance (if any).
7558	*/
7559	VMMRZCallRing3Disable(pVCpu);
7560
7561	/*
7562	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7563	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7564	*
7565	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7566	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7567	*
7568	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7569	* executing guest code.
7570	*/
7571	pVmxTransient->uEflags = ASMIntDisableFlags();
7572	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7573	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7574	{
7575	hmR0VmxClearEventVmcs(pVCpu);
7576	ASMSetFlags(pVmxTransient->uEflags);
7577	VMMRZCallRing3Enable(pVCpu);
7578	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7579	return VINF_EM_RAW_TO_R3;
7580	}
7581	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7582	{
7583	hmR0VmxClearEventVmcs(pVCpu);
7584	ASMSetFlags(pVmxTransient->uEflags);
7585	VMMRZCallRing3Enable(pVCpu);
7586	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7587	return VINF_EM_RAW_INTERRUPT;
7588	}
7589
7590	/* We've injected any pending events. This is really the point of no return (to ring-3). */
7591	pVCpu->hm.s.Event.fPending = false;
7592
7593	return VINF_SUCCESS;
7594	}
7595
7596
7597	/**
7598	* Prepares to run guest code in VT-x and we've committed to doing so. This
7599	* means there is no backing out to ring-3 or anywhere else at this
7600	* point.
7601	*
7602	* @param pVM Pointer to the VM.
7603	* @param pVCpu Pointer to the VMCPU.
7604	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7605	* out-of-sync. Make sure to update the required fields
7606	* before using them.
7607	* @param pVmxTransient Pointer to the VMX transient structure.
7608	*
7609	* @remarks Called with preemption disabled.
7610	* @remarks No-long-jump zone!!!
7611	*/
7612	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7613	{
7614	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7615	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7616	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7617
7618	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7619	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7620
7621	/*
7622	* If we are injecting events to a real-on-v86 mode guest, we may have to update
7623	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7624	* Reload only the necessary state, the assertion will catch if other parts of the code
7625	* change.
7626	*/
7627	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7628	{
7629	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7630	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7631	}
7632
7633	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7634	if (!CPUMIsGuestFPUStateActive(pVCpu))
7635	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7636	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7637	#endif
7638
7639	/*
7640	* Load the host state bits as we may've been preempted (only happens when
7641	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7642	*/
7643	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7644	{
7645	/* This ASSUMES that pfnStartVM has been set up already. */
7646	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7647	AssertRC(rc);
7648	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7649	}
7650	Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
7651
7652	/*
7653	* Load the state shared between host and guest (FPU, debug).
7654	*/
7655	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
7656	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7657	AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7658
7659	/* Store status of the shared guest-host state at the time of VM-entry. */
7660	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7661	if (CPUMIsGuestInLongModeEx(pMixedCtx))
7662	{
7663	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7664	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7665	}
7666	else
7667	#endif
7668	{
7669	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7670	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7671	}
7672	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7673
7674	/*
7675	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7676	*/
7677	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7678	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7679
7680	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7681	RTCPUID idCurrentCpu = pCpu->idCpu;
7682	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7683	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
7684	{
7685	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7686	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7687	}
7688
7689	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7690	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7691	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7692	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7693
7694	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7695
7696	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7697	to start executing. */
7698
7699	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7700	/*
7701	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7702	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7703	*/
7704	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7705	{
7706	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7707	uint64_t u64HostTscAux = 0;
7708	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7709	AssertRC(rc2);
7710	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7711	}
7712	#endif
7713	}
7714
7715
7716	/**
7717	* Performs some essential restoration of state after running guest code in
7718	* VT-x.
7719	*
7720	* @param pVM Pointer to the VM.
7721	* @param pVCpu Pointer to the VMCPU.
7722	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7723	* out-of-sync. Make sure to update the required fields
7724	* before using them.
7725	* @param pVmxTransient Pointer to the VMX transient structure.
7726	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7727	*
7728	* @remarks Called with interrupts disabled, and returns with interrups enabled!
7729	*
7730	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7731	* unconditionally when it is safe to do so.
7732	*/
7733	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7734	{
7735	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7736
7737	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7738	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7739	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7740	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7741	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7742
7743	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7744	{
7745	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7746	/* Restore host's TSC_AUX. */
7747	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7748	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7749	#endif
7750	/** @todo Find a way to fix hardcoding a guestimate. */
7751	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7752	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7753	}
7754
7755	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7756	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7757	Assert(!(ASMGetFlags() & X86_EFL_IF));
7758	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7759
7760	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7761	if (CPUMIsGuestFPUStateActive(pVCpu))
7762	{
7763	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7764	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7765	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7766	}
7767	#endif
7768
7769	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7770	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7771	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7772	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7773
7774	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7775	uint32_t uExitReason;
7776	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7777	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
7778	AssertRC(rc);
7779	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7780	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntInfo);
7781
7782	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7783	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7784	{
7785	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7786	pVmxTransient->fVMEntryFailed));
7787	return;
7788	}
7789
7790	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7791	{
7792	/* Update the guest interruptibility-state from the VMCS. */
7793	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7794	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7795	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7796	AssertRC(rc);
7797	#endif
7798	/*
7799	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7800	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7801	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7802	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7803	*/
7804	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7805	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7806	{
7807	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7808	AssertRC(rc);
7809	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
7810	}
7811	}
7812	}
7813
7814
7815
7816	/**
7817	* Runs the guest code using VT-x the normal way.
7818	*
7819	* @returns VBox status code.
7820	* @param pVM Pointer to the VM.
7821	* @param pVCpu Pointer to the VMCPU.
7822	* @param pCtx Pointer to the guest-CPU context.
7823	*
7824	* @note Mostly the same as hmR0VmxRunGuestCodeStep.
7825	* @remarks Called with preemption disabled.
7826	*/
7827	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7828	{
7829	VMXTRANSIENT VmxTransient;
7830	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7831	int rc = VERR_INTERNAL_ERROR_5;
7832	uint32_t cLoops = 0;
7833
7834	for (;; cLoops++)
7835	{
7836	Assert(!HMR0SuspendPending());
7837	HMVMX_ASSERT_CPU_SAFE();
7838
7839	/* Preparatory work for running guest code, this may force us to return
7840	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7841	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7842	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7843	if (rc != VINF_SUCCESS)
7844	break;
7845
7846	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7847	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7848	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7849
7850	/* Restore any residual host-state and save any bits shared between host
7851	and guest into the guest-CPU state. Re-enables interrupts! */
7852	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7853
7854	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7855	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7856	{
7857	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7858	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7859	return rc;
7860	}
7861
7862	/* Handle the VM-exit. */
7863	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7864	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7865	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7866	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7867	HMVMX_START_EXIT_DISPATCH_PROF();
7868	#ifdef HMVMX_USE_FUNCTION_TABLE
7869	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7870	#else
7871	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7872	#endif
7873	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7874	if (rc != VINF_SUCCESS)
7875	break;
7876	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7877	{
7878	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7879	rc = VINF_EM_RAW_INTERRUPT;
7880	break;
7881	}
7882	}
7883
7884	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7885	return rc;
7886	}
7887
7888
7889	/**
7890	* Single steps guest code using VT-x.
7891	*
7892	* @returns VBox status code.
7893	* @param pVM Pointer to the VM.
7894	* @param pVCpu Pointer to the VMCPU.
7895	* @param pCtx Pointer to the guest-CPU context.
7896	*
7897	* @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7898	* @remarks Called with preemption disabled.
7899	*/
7900	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7901	{
7902	VMXTRANSIENT VmxTransient;
7903	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7904	int rc = VERR_INTERNAL_ERROR_5;
7905	uint32_t cLoops = 0;
7906	uint16_t uCsStart = pCtx->cs.Sel;
7907	uint64_t uRipStart = pCtx->rip;
7908
7909	for (;; cLoops++)
7910	{
7911	Assert(!HMR0SuspendPending());
7912	HMVMX_ASSERT_CPU_SAFE();
7913
7914	/* Preparatory work for running guest code, this may force us to return
7915	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7916	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7917	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7918	if (rc != VINF_SUCCESS)
7919	break;
7920
7921	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7922	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7923	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7924
7925	/* Restore any residual host-state and save any bits shared between host
7926	and guest into the guest-CPU state. Re-enables interrupts! */
7927	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7928
7929	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7930	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7931	{
7932	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7933	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7934	return rc;
7935	}
7936
7937	/* Handle the VM-exit. */
7938	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7939	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7940	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7941	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7942	HMVMX_START_EXIT_DISPATCH_PROF();
7943	#ifdef HMVMX_USE_FUNCTION_TABLE
7944	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7945	#else
7946	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7947	#endif
7948	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7949	if (rc != VINF_SUCCESS)
7950	break;
7951	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7952	{
7953	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7954	rc = VINF_EM_RAW_INTERRUPT;
7955	break;
7956	}
7957
7958	/*
7959	* Did the RIP change, if so, consider it a single step.
7960	* Otherwise, make sure one of the TFs gets set.
7961	*/
7962	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7963	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7964	AssertRCReturn(rc2, rc2);
7965	if ( pCtx->rip != uRipStart
7966	\|\| pCtx->cs.Sel != uCsStart)
7967	{
7968	rc = VINF_EM_DBG_STEPPED;
7969	break;
7970	}
7971	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7972	}
7973
7974	/*
7975	* Clear the X86_EFL_TF if necessary.
7976	*/
7977	if (pVCpu->hm.s.fClearTrapFlag)
7978	{
7979	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
7980	AssertRCReturn(rc2, rc2);
7981	pVCpu->hm.s.fClearTrapFlag = false;
7982	pCtx->eflags.Bits.u1TF = 0;
7983	}
7984	/** @todo there seems to be issues with the resume flag when the monitor trap
7985	* flag is pending without being used. Seen early in bios init when
7986	* accessing APIC page in prot mode. */
7987
7988	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7989	return rc;
7990	}
7991
7992
7993	/**
7994	* Runs the guest code using VT-x.
7995	*
7996	* @returns VBox status code.
7997	* @param pVM Pointer to the VM.
7998	* @param pVCpu Pointer to the VMCPU.
7999	* @param pCtx Pointer to the guest-CPU context.
8000	*
8001	* @remarks Called with preemption disabled.
8002	*/
8003	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8004	{
8005	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8006	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8007	HMVMX_ASSERT_PREEMPT_SAFE();
8008
8009	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8010
8011	int rc;
8012	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8013	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8014	else
8015	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8016
8017	if (rc == VERR_EM_INTERPRETER)
8018	rc = VINF_EM_RAW_EMULATE_INSTR;
8019	else if (rc == VINF_EM_RESET)
8020	rc = VINF_EM_TRIPLE_FAULT;
8021
8022	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8023	if (RT_FAILURE(rc2))
8024	{
8025	pVCpu->hm.s.u32HMError = rc;
8026	rc = rc2;
8027	}
8028	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8029	return rc;
8030	}
8031
8032
8033	#ifndef HMVMX_USE_FUNCTION_TABLE
8034	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8035	{
8036	int rc;
8037	switch (rcReason)
8038	{
8039	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
8040	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
8041	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
8042	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
8043	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
8044	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
8045	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8046	case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
8047	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
8048	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
8049	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8050	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
8051	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
8052	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
8053	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
8054	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8055	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8056	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
8057	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
8058	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
8059	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
8060	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
8061	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
8062	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
8063	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
8064	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8065	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8066	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
8067	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
8068	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
8069	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
8070	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
8071	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
8072
8073	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8074	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8075	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8076	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8077	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8078	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8079	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8080	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8081	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8082
8083	case VMX_EXIT_VMCALL:
8084	case VMX_EXIT_VMCLEAR:
8085	case VMX_EXIT_VMLAUNCH:
8086	case VMX_EXIT_VMPTRLD:
8087	case VMX_EXIT_VMPTRST:
8088	case VMX_EXIT_VMREAD:
8089	case VMX_EXIT_VMRESUME:
8090	case VMX_EXIT_VMWRITE:
8091	case VMX_EXIT_VMXOFF:
8092	case VMX_EXIT_VMXON:
8093	case VMX_EXIT_INVEPT:
8094	case VMX_EXIT_INVVPID:
8095	case VMX_EXIT_VMFUNC:
8096	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8097	break;
8098	default:
8099	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8100	break;
8101	}
8102	return rc;
8103	}
8104	#endif
8105
8106	#ifdef DEBUG
8107	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8108	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8109	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8110
8111	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8112	do \
8113	{ \
8114	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8115	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8116	} while (0)
8117
8118	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8119	do { \
8120	AssertPtr(pVCpu); \
8121	AssertPtr(pMixedCtx); \
8122	AssertPtr(pVmxTransient); \
8123	Assert(pVmxTransient->fVMEntryFailed == false); \
8124	Assert(ASMIntAreEnabled()); \
8125	HMVMX_ASSERT_PREEMPT_SAFE(); \
8126	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8127	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8128	HMVMX_ASSERT_PREEMPT_SAFE(); \
8129	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8130	HMVMX_ASSERT_PREEMPT_CPUID(); \
8131	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8132	} while (0)
8133
8134	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8135	do { \
8136	Log4Func(("\n")); \
8137	} while(0)
8138	#else /* Release builds */
8139	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8140	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8141	#endif
8142
8143
8144	/**
8145	* Advances the guest RIP after reading it from the VMCS.
8146	*
8147	* @returns VBox status code.
8148	* @param pVCpu Pointer to the VMCPU.
8149	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8150	* out-of-sync. Make sure to update the required fields
8151	* before using them.
8152	* @param pVmxTransient Pointer to the VMX transient structure.
8153	*
8154	* @remarks No-long-jump zone!!!
8155	*/
8156	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8157	{
8158	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8159	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8160	AssertRCReturn(rc, rc);
8161
8162	pMixedCtx->rip += pVmxTransient->cbInstr;
8163	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8164	return rc;
8165	}
8166
8167
8168	/**
8169	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8170	* and update error record fields accordingly.
8171	*
8172	* @return VMX_IGS_* return codes.
8173	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8174	* wrong with the guest state.
8175	*
8176	* @param pVM Pointer to the VM.
8177	* @param pVCpu Pointer to the VMCPU.
8178	* @param pCtx Pointer to the guest-CPU state.
8179	*/
8180	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8181	{
8182	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8183	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8184	uError = (err); \
8185	break; \
8186	} else do {} while (0)
8187	/* Duplicate of IEM_IS_CANONICAL(). */
8188	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8189
8190	int rc;
8191	uint32_t uError = VMX_IGS_ERROR;
8192	uint32_t u32Val;
8193	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8194
8195	do
8196	{
8197	/*
8198	* CR0.
8199	*/
8200	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8201	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8202	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8203	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8204	if (fUnrestrictedGuest)
8205	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8206
8207	uint32_t u32GuestCR0;
8208	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8209	AssertRCBreak(rc);
8210	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8211	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8212	if ( !fUnrestrictedGuest
8213	&& (u32GuestCR0 & X86_CR0_PG)
8214	&& !(u32GuestCR0 & X86_CR0_PE))
8215	{
8216	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8217	}
8218
8219	/*
8220	* CR4.
8221	*/
8222	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8223	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8224
8225	uint32_t u32GuestCR4;
8226	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8227	AssertRCBreak(rc);
8228	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8229	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8230
8231	/*
8232	* IA32_DEBUGCTL MSR.
8233	*/
8234	uint64_t u64Val;
8235	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8236	AssertRCBreak(rc);
8237	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8238	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8239	{
8240	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8241	}
8242	uint64_t u64DebugCtlMsr = u64Val;
8243
8244	#ifdef VBOX_STRICT
8245	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8246	AssertRCBreak(rc);
8247	Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
8248	#endif
8249	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8250
8251	/*
8252	* RIP and RFLAGS.
8253	*/
8254	uint32_t u32Eflags;
8255	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8256	if (HMVMX_IS_64BIT_HOST_MODE())
8257	{
8258	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8259	AssertRCBreak(rc);
8260	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8261	if ( !fLongModeGuest
8262	\|\| !pCtx->cs.Attr.n.u1Long)
8263	{
8264	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8265	}
8266	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8267	* must be identical if the "IA32e mode guest" VM-entry control is 1
8268	* and CS.L is 1. No check applies if the CPU supports 64
8269	* linear-address bits. */
8270
8271	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8272	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8273	AssertRCBreak(rc);
8274	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8275	VMX_IGS_RFLAGS_RESERVED);
8276	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8277	u32Eflags = u64Val;
8278	}
8279	else
8280	#endif
8281	{
8282	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8283	AssertRCBreak(rc);
8284	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8285	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8286	}
8287
8288	if ( fLongModeGuest
8289	\|\| ( fUnrestrictedGuest
8290	&& !(u32GuestCR0 & X86_CR0_PE)))
8291	{
8292	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8293	}
8294
8295	uint32_t u32EntryInfo;
8296	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8297	AssertRCBreak(rc);
8298	if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8299	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8300	{
8301	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8302	}
8303
8304	/*
8305	* 64-bit checks.
8306	*/
8307	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8308	if (HMVMX_IS_64BIT_HOST_MODE())
8309	{
8310	if ( fLongModeGuest
8311	&& !fUnrestrictedGuest)
8312	{
8313	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8314	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8315	}
8316
8317	if ( !fLongModeGuest
8318	&& (u32GuestCR4 & X86_CR4_PCIDE))
8319	{
8320	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8321	}
8322
8323	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8324	* 51:32 beyond the processor's physical-address width are 0. */
8325
8326	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8327	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8328	{
8329	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8330	}
8331
8332	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8333	AssertRCBreak(rc);
8334	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8335
8336	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8337	AssertRCBreak(rc);
8338	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8339	}
8340	#endif
8341
8342	/*
8343	* PERF_GLOBAL MSR.
8344	*/
8345	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8346	{
8347	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8348	AssertRCBreak(rc);
8349	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8350	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8351	}
8352
8353	/*
8354	* PAT MSR.
8355	*/
8356	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8357	{
8358	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8359	AssertRCBreak(rc);
8360	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8361	for (unsigned i = 0; i < 8; i++)
8362	{
8363	uint8_t u8Val = (u64Val & 0x7);
8364	if ( u8Val != 0 /* UC */
8365	\|\| u8Val != 1 /* WC */
8366	\|\| u8Val != 4 /* WT */
8367	\|\| u8Val != 5 /* WP */
8368	\|\| u8Val != 6 /* WB */
8369	\|\| u8Val != 7 /* UC- */)
8370	{
8371	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8372	}
8373	u64Val >>= 3;
8374	}
8375	}
8376
8377	/*
8378	* EFER MSR.
8379	*/
8380	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8381	{
8382	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8383	AssertRCBreak(rc);
8384	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8385	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8386	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8387	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8388	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8389	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8390	}
8391
8392	/*
8393	* Segment registers.
8394	*/
8395	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8396	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8397	if (!(u32Eflags & X86_EFL_VM))
8398	{
8399	/* CS */
8400	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8401	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8402	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8403	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8404	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8405	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8406	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8407	/* CS cannot be loaded with NULL in protected mode. */
8408	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8409	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8410	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8411	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8412	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8413	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8414	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8415	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8416	else
8417	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8418
8419	/* SS */
8420	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8421	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8422	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8423	if ( !(pCtx->cr0 & X86_CR0_PE)
8424	\|\| pCtx->cs.Attr.n.u4Type == 3)
8425	{
8426	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8427	}
8428	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8429	{
8430	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8431	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8432	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8433	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8434	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8435	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8436	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8437	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8438	}
8439
8440	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8441	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8442	{
8443	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8444	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8445	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8446	\|\| pCtx->ds.Attr.n.u4Type > 11
8447	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8448	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8449	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8450	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8451	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8452	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8453	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8454	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8455	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8456	}
8457	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8458	{
8459	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8460	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8461	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8462	\|\| pCtx->es.Attr.n.u4Type > 11
8463	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8464	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8465	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8466	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8467	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8468	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8469	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8470	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8471	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8472	}
8473	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8474	{
8475	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8476	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8477	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8478	\|\| pCtx->fs.Attr.n.u4Type > 11
8479	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8480	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8481	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8482	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8483	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8484	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8485	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8486	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8487	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8488	}
8489	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8490	{
8491	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8492	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8493	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8494	\|\| pCtx->gs.Attr.n.u4Type > 11
8495	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8496	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8497	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8498	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8499	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8500	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8501	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8502	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8503	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8504	}
8505	/* 64-bit capable CPUs. */
8506	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8507	if (HMVMX_IS_64BIT_HOST_MODE())
8508	{
8509	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8510	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8511	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8512	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8513	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8514	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8515	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8516	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8517	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8518	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8519	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8520	}
8521	#endif
8522	}
8523	else
8524	{
8525	/* V86 mode checks. */
8526	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8527	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8528	{
8529	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8530	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8531	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8532	}
8533	else
8534	{
8535	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8536	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8537	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8538	}
8539
8540	/* CS */
8541	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8542	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8543	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8544	/* SS */
8545	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8546	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8547	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8548	/* DS */
8549	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8550	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8551	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8552	/* ES */
8553	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8554	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8555	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8556	/* FS */
8557	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8558	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8559	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8560	/* GS */
8561	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8562	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8563	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8564	/* 64-bit capable CPUs. */
8565	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8566	if (HMVMX_IS_64BIT_HOST_MODE())
8567	{
8568	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8569	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8570	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8571	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8572	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8573	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8574	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8575	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8576	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8577	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8578	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8579	}
8580	#endif
8581	}
8582
8583	/*
8584	* TR.
8585	*/
8586	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8587	/* 64-bit capable CPUs. */
8588	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8589	if (HMVMX_IS_64BIT_HOST_MODE())
8590	{
8591	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8592	}
8593	#endif
8594	if (fLongModeGuest)
8595	{
8596	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8597	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8598	}
8599	else
8600	{
8601	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8602	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8603	VMX_IGS_TR_ATTR_TYPE_INVALID);
8604	}
8605	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8606	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8607	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8608	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8609	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8610	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8611	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8612	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8613
8614	/*
8615	* GDTR and IDTR.
8616	*/
8617	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8618	if (HMVMX_IS_64BIT_HOST_MODE())
8619	{
8620	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8621	AssertRCBreak(rc);
8622	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8623
8624	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8625	AssertRCBreak(rc);
8626	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8627	}
8628	#endif
8629
8630	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8631	AssertRCBreak(rc);
8632	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8633
8634	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8635	AssertRCBreak(rc);
8636	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8637
8638	/*
8639	* Guest Non-Register State.
8640	*/
8641	/* Activity State. */
8642	uint32_t u32ActivityState;
8643	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8644	AssertRCBreak(rc);
8645	HMVMX_CHECK_BREAK( !u32ActivityState
8646	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8647	VMX_IGS_ACTIVITY_STATE_INVALID);
8648	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8649	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8650	uint32_t u32IntrState;
8651	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8652	AssertRCBreak(rc);
8653	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8654	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8655	{
8656	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8657	}
8658
8659	/** @todo Activity state and injecting interrupts. Left as a todo since we
8660	* currently don't use activity states but ACTIVE. */
8661
8662	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8663	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8664
8665	/* Guest interruptibility-state. */
8666	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8667	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8668	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8669	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8670	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8671	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8672	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8673	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8674	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8675	if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8676	{
8677	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8678	{
8679	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8680	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8681	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8682	}
8683	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8684	{
8685	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8686	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8687	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8688	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8689	}
8690	}
8691	/** @todo Assumes the processor is not in SMM. */
8692	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8693	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8694	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8695	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8696	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8697	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8698	&& VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8699	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8700	{
8701	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8702	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8703	}
8704
8705	/* Pending debug exceptions. */
8706	if (HMVMX_IS_64BIT_HOST_MODE())
8707	{
8708	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8709	AssertRCBreak(rc);
8710	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8711	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8712	u32Val = u64Val; /* For pending debug exceptions checks below. */
8713	}
8714	else
8715	{
8716	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8717	AssertRCBreak(rc);
8718	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8719	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8720	}
8721
8722	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8723	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8724	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8725	{
8726	if ( (u32Eflags & X86_EFL_TF)
8727	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8728	{
8729	/* Bit 14 is PendingDebug.BS. */
8730	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8731	}
8732	if ( !(u32Eflags & X86_EFL_TF)
8733	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8734	{
8735	/* Bit 14 is PendingDebug.BS. */
8736	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8737	}
8738	}
8739
8740	/* VMCS link pointer. */
8741	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8742	AssertRCBreak(rc);
8743	if (u64Val != UINT64_C(0xffffffffffffffff))
8744	{
8745	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8746	/** @todo Bits beyond the processor's physical-address width MBZ. */
8747	/** @todo 32-bit located in memory referenced by value of this field (as a
8748	* physical address) must contain the processor's VMCS revision ID. */
8749	/** @todo SMM checks. */
8750	}
8751
8752	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8753
8754	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8755	if (uError == VMX_IGS_ERROR)
8756	uError = VMX_IGS_REASON_NOT_FOUND;
8757	} while (0);
8758
8759	pVCpu->hm.s.u32HMError = uError;
8760	return uError;
8761
8762	#undef HMVMX_ERROR_BREAK
8763	#undef HMVMX_CHECK_BREAK
8764	#undef HMVMX_IS_CANONICAL
8765	}
8766
8767	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8768	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8769	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8770
8771	/** @name VM-exit handlers.
8772	* @{
8773	*/
8774
8775	/**
8776	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8777	*/
8778	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8779	{
8780	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8781	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8782	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8783	#if HC_ARCH_BITS == 64
8784	Assert(ASMIntAreEnabled());
8785	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8786	return VINF_SUCCESS;
8787	#endif
8788	return VINF_EM_RAW_INTERRUPT;
8789	}
8790
8791
8792	/**
8793	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8794	*/
8795	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8796	{
8797	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8798	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8799
8800	int rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
8801	AssertRCReturn(rc, rc);
8802
8803	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
8804	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8805	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8806	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
8807
8808	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8809	{
8810	/*
8811	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8812	* anything we inject is not going to cause a VM-exit directly for the event being injected.
8813	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8814	*
8815	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8816	*/
8817	VMXDispatchHostNmi();
8818	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
8819	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8820	return VINF_SUCCESS;
8821	}
8822
8823	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8824	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8825	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8826	{
8827	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8828	return VINF_SUCCESS;
8829	}
8830	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8831	{
8832	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8833	return rc;
8834	}
8835
8836	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
8837	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
8838	switch (uIntType)
8839	{
8840	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8841	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8842	/* no break */
8843	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8844	{
8845	switch (uVector)
8846	{
8847	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8848	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8849	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8850	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8851	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8852	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8853	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8854	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8855	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8856	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8857	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8858	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8859	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8860	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8861	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8862	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8863	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8864	#endif
8865	default:
8866	{
8867	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8868	AssertRCReturn(rc, rc);
8869
8870	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8871	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8872	{
8873	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8874	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8875	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8876
8877	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8878	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
8879	AssertRCReturn(rc, rc);
8880	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
8881	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
8882	0 /* GCPtrFaultAddress */);
8883	AssertRCReturn(rc, rc);
8884	}
8885	else
8886	{
8887	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8888	pVCpu->hm.s.u32HMError = uVector;
8889	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8890	}
8891	break;
8892	}
8893	}
8894	break;
8895	}
8896
8897	default:
8898	{
8899	pVCpu->hm.s.u32HMError = uExitIntInfo;
8900	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
8901	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
8902	break;
8903	}
8904	}
8905	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8906	return rc;
8907	}
8908
8909
8910	/**
8911	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8912	*/
8913	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8914	{
8915	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8916
8917	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8918	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8919	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8920	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8921	AssertRCReturn(rc, rc);
8922
8923	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8924	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8925	return VINF_SUCCESS;
8926	}
8927
8928
8929	/**
8930	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8931	*/
8932	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8933	{
8934	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8935	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8936	HMVMX_RETURN_UNEXPECTED_EXIT();
8937	}
8938
8939
8940	/**
8941	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8942	*/
8943	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8944	{
8945	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8946	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8947	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8948	}
8949
8950
8951	/**
8952	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8953	*/
8954	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8955	{
8956	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8957	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8958	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8959	}
8960
8961
8962	/**
8963	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8964	*/
8965	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8966	{
8967	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8968	PVM pVM = pVCpu->CTX_SUFF(pVM);
8969	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8970	if (RT_LIKELY(rc == VINF_SUCCESS))
8971	{
8972	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8973	Assert(pVmxTransient->cbInstr == 2);
8974	}
8975	else
8976	{
8977	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8978	rc = VERR_EM_INTERPRETER;
8979	}
8980	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8981	return rc;
8982	}
8983
8984
8985	/**
8986	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8987	*/
8988	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8989	{
8990	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8991	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8992	AssertRCReturn(rc, rc);
8993
8994	if (pMixedCtx->cr4 & X86_CR4_SMXE)
8995	return VINF_EM_RAW_EMULATE_INSTR;
8996
8997	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8998	HMVMX_RETURN_UNEXPECTED_EXIT();
8999	}
9000
9001
9002	/**
9003	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9004	*/
9005	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9006	{
9007	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9008	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9009	AssertRCReturn(rc, rc);
9010
9011	PVM pVM = pVCpu->CTX_SUFF(pVM);
9012	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9013	if (RT_LIKELY(rc == VINF_SUCCESS))
9014	{
9015	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9016	Assert(pVmxTransient->cbInstr == 2);
9017	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9018	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9019	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9020	}
9021	else
9022	{
9023	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9024	rc = VERR_EM_INTERPRETER;
9025	}
9026	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9027	return rc;
9028	}
9029
9030
9031	/**
9032	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9033	*/
9034	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9035	{
9036	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9037	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9038	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9039	AssertRCReturn(rc, rc);
9040
9041	PVM pVM = pVCpu->CTX_SUFF(pVM);
9042	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9043	if (RT_LIKELY(rc == VINF_SUCCESS))
9044	{
9045	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9046	Assert(pVmxTransient->cbInstr == 3);
9047	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9048	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9049	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9050	}
9051	else
9052	{
9053	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9054	rc = VERR_EM_INTERPRETER;
9055	}
9056	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9057	return rc;
9058	}
9059
9060
9061	/**
9062	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9063	*/
9064	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9065	{
9066	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9067	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9068	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9069	AssertRCReturn(rc, rc);
9070
9071	PVM pVM = pVCpu->CTX_SUFF(pVM);
9072	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9073	if (RT_LIKELY(rc == VINF_SUCCESS))
9074	{
9075	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9076	Assert(pVmxTransient->cbInstr == 2);
9077	}
9078	else
9079	{
9080	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9081	rc = VERR_EM_INTERPRETER;
9082	}
9083	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9084	return rc;
9085	}
9086
9087
9088	/**
9089	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9090	*/
9091	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9092	{
9093	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9094	PVM pVM = pVCpu->CTX_SUFF(pVM);
9095	Assert(!pVM->hm.s.fNestedPaging);
9096
9097	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9098	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9099	AssertRCReturn(rc, rc);
9100
9101	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9102	rc = VBOXSTRICTRC_VAL(rc2);
9103	if (RT_LIKELY(rc == VINF_SUCCESS))
9104	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9105	else
9106	{
9107	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9108	pVmxTransient->uExitQualification, rc));
9109	}
9110	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9111	return rc;
9112	}
9113
9114
9115	/**
9116	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9117	*/
9118	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9119	{
9120	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9121	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9122	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9123	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9124	AssertRCReturn(rc, rc);
9125
9126	PVM pVM = pVCpu->CTX_SUFF(pVM);
9127	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9128	if (RT_LIKELY(rc == VINF_SUCCESS))
9129	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9130	else
9131	{
9132	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9133	rc = VERR_EM_INTERPRETER;
9134	}
9135	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9136	return rc;
9137	}
9138
9139
9140	/**
9141	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9142	*/
9143	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9144	{
9145	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9146	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9147	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9148	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9149	AssertRCReturn(rc, rc);
9150
9151	PVM pVM = pVCpu->CTX_SUFF(pVM);
9152	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9153	rc = VBOXSTRICTRC_VAL(rc2);
9154	if (RT_LIKELY( rc == VINF_SUCCESS
9155	\|\| rc == VINF_EM_HALT))
9156	{
9157	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9158	AssertRCReturn(rc3, rc3);
9159
9160	if ( rc == VINF_EM_HALT
9161	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9162	{
9163	rc = VINF_SUCCESS;
9164	}
9165	}
9166	else
9167	{
9168	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9169	rc = VERR_EM_INTERPRETER;
9170	}
9171	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9172	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9173	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9174	return rc;
9175	}
9176
9177
9178	/**
9179	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9180	*/
9181	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9182	{
9183	/*
9184	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9185	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9186	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9187	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9188	*/
9189	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9190	HMVMX_RETURN_UNEXPECTED_EXIT();
9191	}
9192
9193
9194	/**
9195	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9196	*/
9197	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9198	{
9199	/*
9200	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9201	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9202	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9203	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9204	*/
9205	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9206	HMVMX_RETURN_UNEXPECTED_EXIT();
9207	}
9208
9209
9210	/**
9211	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9212	*/
9213	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9214	{
9215	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9216	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9217	HMVMX_RETURN_UNEXPECTED_EXIT();
9218	}
9219
9220
9221	/**
9222	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9223	*/
9224	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9225	{
9226	/*
9227	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9228	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9229	* See Intel spec. 25.3 "Other Causes of VM-exits".
9230	*/
9231	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9232	HMVMX_RETURN_UNEXPECTED_EXIT();
9233	}
9234
9235
9236	/**
9237	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9238	* VM-exit.
9239	*/
9240	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9241	{
9242	/*
9243	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9244	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9245	*
9246	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9247	* See Intel spec. "23.8 Restrictions on VMX operation".
9248	*/
9249	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9250	return VINF_SUCCESS;
9251	}
9252
9253
9254	/**
9255	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9256	* VM-exit.
9257	*/
9258	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9259	{
9260	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9261	return VINF_EM_RESET;
9262	}
9263
9264
9265	/**
9266	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9267	*/
9268	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9269	{
9270	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9271	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9272	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9273	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9274	AssertRCReturn(rc, rc);
9275
9276	pMixedCtx->rip++;
9277	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9278	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9279	rc = VINF_SUCCESS;
9280	else
9281	rc = VINF_EM_HALT;
9282
9283	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9284	return rc;
9285	}
9286
9287
9288	/**
9289	* VM-exit handler for instructions that result in a #UD exception delivered to
9290	* the guest.
9291	*/
9292	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9293	{
9294	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9295	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9296	return VINF_SUCCESS;
9297	}
9298
9299
9300	/**
9301	* VM-exit handler for expiry of the VMX preemption timer.
9302	*/
9303	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9304	{
9305	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9306
9307	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9308	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9309
9310	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9311	PVM pVM = pVCpu->CTX_SUFF(pVM);
9312	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9313	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9314	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9315	}
9316
9317
9318	/**
9319	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9320	*/
9321	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9322	{
9323	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9324
9325	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9326	/** @todo check if XSETBV is supported by the recompiler. */
9327	return VERR_EM_INTERPRETER;
9328	}
9329
9330
9331	/**
9332	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9333	*/
9334	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9335	{
9336	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9337
9338	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9339	/** @todo implement EMInterpretInvpcid() */
9340	return VERR_EM_INTERPRETER;
9341	}
9342
9343
9344	/**
9345	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9346	* Error VM-exit.
9347	*/
9348	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9349	{
9350	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9351	AssertRCReturn(rc, rc);
9352
9353	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9354	NOREF(uInvalidReason);
9355
9356	#ifdef VBOX_STRICT
9357	uint32_t uIntrState;
9358	HMVMXHCUINTREG uHCReg;
9359	uint64_t u64Val;
9360	uint32_t u32Val;
9361
9362	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9363	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9364	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9365	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9366	AssertRCReturn(rc, rc);
9367
9368	Log4(("uInvalidReason %u\n", uInvalidReason));
9369	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9370	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9371	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9372	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9373
9374	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9375	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9376	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9377	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9378	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9379	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9380	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9381	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9382	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9383	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9384	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9385	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9386	#endif
9387
9388	PVM pVM = pVCpu->CTX_SUFF(pVM);
9389	HMDumpRegs(pVM, pVCpu, pMixedCtx);
9390
9391	return VERR_VMX_INVALID_GUEST_STATE;
9392	}
9393
9394
9395	/**
9396	* VM-exit handler for VM-entry failure due to an MSR-load
9397	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9398	*/
9399	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9400	{
9401	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9402	HMVMX_RETURN_UNEXPECTED_EXIT();
9403	}
9404
9405
9406	/**
9407	* VM-exit handler for VM-entry failure due to a machine-check event
9408	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9409	*/
9410	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9411	{
9412	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9413	HMVMX_RETURN_UNEXPECTED_EXIT();
9414	}
9415
9416
9417	/**
9418	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9419	* theory.
9420	*/
9421	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9422	{
9423	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9424	return VERR_VMX_UNDEFINED_EXIT_CODE;
9425	}
9426
9427
9428	/**
9429	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9430	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9431	* Conditional VM-exit.
9432	*/
9433	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9434	{
9435	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9436
9437	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9438	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9439	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9440	return VERR_EM_INTERPRETER;
9441	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9442	HMVMX_RETURN_UNEXPECTED_EXIT();
9443	}
9444
9445
9446	/**
9447	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9448	*/
9449	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9450	{
9451	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9452
9453	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9454	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9455	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9456	return VERR_EM_INTERPRETER;
9457	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9458	HMVMX_RETURN_UNEXPECTED_EXIT();
9459	}
9460
9461
9462	/**
9463	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9464	*/
9465	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9466	{
9467	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9468
9469	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9470	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9471	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9472	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9473	AssertRCReturn(rc, rc);
9474	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9475
9476	PVM pVM = pVCpu->CTX_SUFF(pVM);
9477	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9478	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9479	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9480	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9481
9482	if (RT_LIKELY(rc == VINF_SUCCESS))
9483	{
9484	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9485	Assert(pVmxTransient->cbInstr == 2);
9486	}
9487	return rc;
9488	}
9489
9490
9491	/**
9492	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9493	*/
9494	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9495	{
9496	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9497	PVM pVM = pVCpu->CTX_SUFF(pVM);
9498	int rc = VINF_SUCCESS;
9499
9500	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9501	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9502	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9503	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9504	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9505	AssertRCReturn(rc, rc);
9506	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9507
9508	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9509	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9510	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9511
9512	if (RT_LIKELY(rc == VINF_SUCCESS))
9513	{
9514	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9515
9516	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9517	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9518	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9519	{
9520	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9521	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9522	EMInterpretWrmsr() changes it. */
9523	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9524	}
9525	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9526	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
9527	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9528	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9529
9530	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9531	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9532	{
9533	switch (pMixedCtx->ecx)
9534	{
9535	case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9536	case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9537	case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9538	case MSR_K8_FS_BASE: /* no break */
9539	case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9540	case MSR_K8_KERNEL_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS); break;
9541	}
9542	}
9543	#ifdef VBOX_STRICT
9544	else
9545	{
9546	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9547	switch (pMixedCtx->ecx)
9548	{
9549	case MSR_IA32_SYSENTER_CS:
9550	case MSR_IA32_SYSENTER_EIP:
9551	case MSR_IA32_SYSENTER_ESP:
9552	case MSR_K8_FS_BASE:
9553	case MSR_K8_GS_BASE:
9554	{
9555	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9556	HMVMX_RETURN_UNEXPECTED_EXIT();
9557	}
9558
9559	case MSR_K8_LSTAR:
9560	case MSR_K6_STAR:
9561	case MSR_K8_SF_MASK:
9562	case MSR_K8_TSC_AUX:
9563	case MSR_K8_KERNEL_GS_BASE:
9564	{
9565	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9566	pMixedCtx->ecx));
9567	HMVMX_RETURN_UNEXPECTED_EXIT();
9568	}
9569	}
9570	}
9571	#endif /* VBOX_STRICT */
9572	}
9573	return rc;
9574	}
9575
9576
9577	/**
9578	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9579	*/
9580	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9581	{
9582	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9583
9584	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9585	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9586	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9587	return VERR_EM_INTERPRETER;
9588	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9589	HMVMX_RETURN_UNEXPECTED_EXIT();
9590	}
9591
9592
9593	/**
9594	* VM-exit handler for when the TPR value is lowered below the specified
9595	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9596	*/
9597	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9598	{
9599	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9600	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9601
9602	/*
9603	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9604	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9605	* resume guest execution.
9606	*/
9607	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9608	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9609	return VINF_SUCCESS;
9610	}
9611
9612
9613	/**
9614	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9615	* VM-exit.
9616	*
9617	* @retval VINF_SUCCESS when guest execution can continue.
9618	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9619	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9620	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9621	* recompiler.
9622	*/
9623	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9624	{
9625	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9626	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9627	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9628	AssertRCReturn(rc, rc);
9629
9630	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9631	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9632	PVM pVM = pVCpu->CTX_SUFF(pVM);
9633	switch (uAccessType)
9634	{
9635	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9636	{
9637	#if 0
9638	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9639	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9640	#else
9641	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9642	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9643	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9644	#endif
9645	AssertRCReturn(rc, rc);
9646
9647	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9648	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9649	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9650	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
9651
9652	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9653	{
9654	case 0: /* CR0 */
9655	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9656	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9657	break;
9658	case 2: /* CR2 */
9659	/* Nothing to do here, CR2 it's not part of the VMCS. */
9660	break;
9661	case 3: /* CR3 */
9662	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9663	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
9664	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9665	break;
9666	case 4: /* CR4 */
9667	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9668	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9669	break;
9670	case 8: /* CR8 */
9671	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9672	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9673	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9674	break;
9675	default:
9676	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9677	break;
9678	}
9679
9680	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9681	break;
9682	}
9683
9684	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9685	{
9686	/* EMInterpretCRxRead() requires EFER MSR, CS. */
9687	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9688	AssertRCReturn(rc, rc);
9689	Assert( !pVM->hm.s.fNestedPaging
9690	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9691	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9692
9693	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9694	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9695	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9696
9697	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9698	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9699	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9700	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9701	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9702	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9703	break;
9704	}
9705
9706	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9707	{
9708	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9709	AssertRCReturn(rc, rc);
9710	rc = EMInterpretCLTS(pVM, pVCpu);
9711	AssertRCReturn(rc, rc);
9712	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9713	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9714	Log4(("CRX CLTS write rc=%d\n", rc));
9715	break;
9716	}
9717
9718	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9719	{
9720	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9721	AssertRCReturn(rc, rc);
9722	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9723	if (RT_LIKELY(rc == VINF_SUCCESS))
9724	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9725	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9726	Log4(("CRX LMSW write rc=%d\n", rc));
9727	break;
9728	}
9729
9730	default:
9731	{
9732	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9733	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9734	}
9735	}
9736
9737	/* Validate possible error codes. */
9738	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
9739	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9740	if (RT_SUCCESS(rc))
9741	{
9742	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9743	AssertRCReturn(rc2, rc2);
9744	}
9745
9746	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9747	return rc;
9748	}
9749
9750
9751	/**
9752	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9753	* VM-exit.
9754	*/
9755	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9756	{
9757	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9758	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9759
9760	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9761	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9762	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9763	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9764	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9765	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9766	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9767	AssertRCReturn(rc2, rc2);
9768
9769	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9770	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9771	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9772	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9773	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9774	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9775	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9776
9777	/* I/O operation lookup arrays. */
9778	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9779	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9780
9781	VBOXSTRICTRC rcStrict;
9782	const uint32_t cbValue = s_aIOSizes[uIOWidth];
9783	const uint32_t cbInstr = pVmxTransient->cbInstr;
9784	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9785	PVM pVM = pVCpu->CTX_SUFF(pVM);
9786	if (fIOString)
9787	{
9788	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158}*/
9789	/*
9790	* INS/OUTS - I/O String instruction.
9791	*
9792	* Use instruction-information if available, otherwise fall back on
9793	* interpreting the instruction.
9794	*/
9795	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9796	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9797	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9798	{
9799	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
9800	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9801	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9802	AssertRCReturn(rc2, rc2);
9803	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9804	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9805	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9806	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9807	if (fIOWrite)
9808	{
9809	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9810	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9811	}
9812	else
9813	{
9814	/*
9815	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
9816	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
9817	* See Intel Instruction spec. for "INS".
9818	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
9819	*/
9820	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9821	}
9822	}
9823	else
9824	{
9825	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9826	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9827	AssertRCReturn(rc2, rc2);
9828	rcStrict = IEMExecOne(pVCpu);
9829	}
9830	/** @todo IEM needs to be setting these flags somehow. */
9831	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9832	fUpdateRipAlready = true;
9833	#else
9834	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9835	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9836	if (RT_SUCCESS(rcStrict))
9837	{
9838	if (fIOWrite)
9839	{
9840	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9841	(DISCPUMODE)pDis->uAddrMode, cbValue);
9842	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9843	}
9844	else
9845	{
9846	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9847	(DISCPUMODE)pDis->uAddrMode, cbValue);
9848	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9849	}
9850	}
9851	else
9852	{
9853	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9854	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9855	}
9856	#endif
9857	}
9858	else
9859	{
9860	/*
9861	* IN/OUT - I/O instruction.
9862	*/
9863	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9864	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9865	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9866	if (fIOWrite)
9867	{
9868	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9869	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9870	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9871	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9872	}
9873	else
9874	{
9875	uint32_t u32Result = 0;
9876	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9877	if (IOM_SUCCESS(rcStrict))
9878	{
9879	/* Save result of I/O IN instr. in AL/AX/EAX. */
9880	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
9881	}
9882	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9883	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9884	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9885	}
9886	}
9887
9888	if (IOM_SUCCESS(rcStrict))
9889	{
9890	if (!fUpdateRipAlready)
9891	{
9892	pMixedCtx->rip += cbInstr;
9893	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9894	}
9895
9896	/*
9897	* If any I/O breakpoints are armed, we need to check if one triggered
9898	* and take appropriate action.
9899	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9900	*/
9901	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9902	AssertRCReturn(rc2, rc2);
9903
9904	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9905	* execution engines about whether hyper BPs and such are pending. */
9906	uint32_t const uDr7 = pMixedCtx->dr[7];
9907	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9908	&& X86_DR7_ANY_RW_IO(uDr7)
9909	&& (pMixedCtx->cr4 & X86_CR4_DE))
9910	\|\| DBGFBpIsHwIoArmed(pVM)))
9911	{
9912	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9913
9914	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
9915	VMMRZCallRing3Disable(pVCpu);
9916	HM_DISABLE_PREEMPT_IF_NEEDED();
9917
9918	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
9919
9920	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9921	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9922	{
9923	/* Raise #DB. */
9924	if (fIsGuestDbgActive)
9925	ASMSetDR6(pMixedCtx->dr[6]);
9926	if (pMixedCtx->dr[7] != uDr7)
9927	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
9928
9929	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9930	}
9931	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9932	else if ( rcStrict2 != VINF_SUCCESS
9933	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
9934	rcStrict = rcStrict2;
9935
9936	HM_RESTORE_PREEMPT_IF_NEEDED();
9937	VMMRZCallRing3Enable(pVCpu);
9938	}
9939	}
9940
9941	#ifdef DEBUG
9942	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9943	Assert(!fIOWrite);
9944	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9945	Assert(fIOWrite);
9946	else
9947	{
9948	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9949	* statuses, that the VMM device and some others may return. See
9950	* IOM_SUCCESS() for guidance. */
9951	AssertMsg( RT_FAILURE(rcStrict)
9952	\|\| rcStrict == VINF_SUCCESS
9953	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
9954	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
9955	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
9956	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9957	}
9958	#endif
9959
9960	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9961	return VBOXSTRICTRC_TODO(rcStrict);
9962	}
9963
9964
9965	/**
9966	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9967	* VM-exit.
9968	*/
9969	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9970	{
9971	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9972
9973	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
9974	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9975	AssertRCReturn(rc, rc);
9976	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9977	{
9978	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9979	AssertRCReturn(rc, rc);
9980	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9981	{
9982	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9983
9984	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9985	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9986	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9987	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9988	{
9989	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9990	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9991
9992	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9993	Assert(!pVCpu->hm.s.Event.fPending);
9994	pVCpu->hm.s.Event.fPending = true;
9995	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
9996	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9997	AssertRCReturn(rc, rc);
9998	if (fErrorCodeValid)
9999	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10000	else
10001	pVCpu->hm.s.Event.u32ErrCode = 0;
10002	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10003	&& uVector == X86_XCPT_PF)
10004	{
10005	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10006	}
10007
10008	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10009	}
10010	}
10011	}
10012
10013	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10014	* emulation. */
10015	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10016	return VERR_EM_INTERPRETER;
10017	}
10018
10019
10020	/**
10021	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10022	*/
10023	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10024	{
10025	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10026	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10027	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10028	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10029	AssertRCReturn(rc, rc);
10030	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10031	return VINF_EM_DBG_STEPPED;
10032	}
10033
10034
10035	/**
10036	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10037	*/
10038	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10039	{
10040	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10041
10042	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10043	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10044	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10045	return VINF_SUCCESS;
10046	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10047	return rc;
10048
10049	#if 0
10050	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10051	* just sync the whole thing. */
10052	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10053	#else
10054	/* Aggressive state sync. for now. */
10055	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10056	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10057	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10058	#endif
10059	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10060	AssertRCReturn(rc, rc);
10061
10062	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10063	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10064	switch (uAccessType)
10065	{
10066	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10067	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10068	{
10069	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10070	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10071	{
10072	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10073	}
10074
10075	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10076	GCPhys &= PAGE_BASE_GC_MASK;
10077	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10078	PVM pVM = pVCpu->CTX_SUFF(pVM);
10079	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10080	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10081
10082	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10083	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10084	CPUMCTX2CORE(pMixedCtx), GCPhys);
10085	rc = VBOXSTRICTRC_VAL(rc2);
10086	Log4(("ApicAccess rc=%d\n", rc));
10087	if ( rc == VINF_SUCCESS
10088	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10089	\|\| rc == VERR_PAGE_NOT_PRESENT)
10090	{
10091	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10092	\| HM_CHANGED_GUEST_RSP
10093	\| HM_CHANGED_GUEST_RFLAGS
10094	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10095	rc = VINF_SUCCESS;
10096	}
10097	break;
10098	}
10099
10100	default:
10101	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10102	rc = VINF_EM_RAW_EMULATE_INSTR;
10103	break;
10104	}
10105
10106	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10107	return rc;
10108	}
10109
10110
10111	/**
10112	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10113	* VM-exit.
10114	*/
10115	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10116	{
10117	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10118
10119	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10120	if (pVmxTransient->fWasGuestDebugStateActive)
10121	{
10122	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10123	HMVMX_RETURN_UNEXPECTED_EXIT();
10124	}
10125
10126	int rc = VERR_INTERNAL_ERROR_5;
10127	if ( !DBGFIsStepping(pVCpu)
10128	&& !pVCpu->hm.s.fSingleInstruction
10129	&& !pVmxTransient->fWasHyperDebugStateActive)
10130	{
10131	/* Don't intercept MOV DRx and #DB any more. */
10132	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10133	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10134	AssertRCReturn(rc, rc);
10135
10136	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10137	{
10138	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10139	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10140	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10141	AssertRCReturn(rc, rc);
10142	#endif
10143	}
10144
10145	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10146	VMMRZCallRing3Disable(pVCpu);
10147	HM_DISABLE_PREEMPT_IF_NEEDED();
10148
10149	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10150	PVM pVM = pVCpu->CTX_SUFF(pVM);
10151	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10152	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10153
10154	HM_RESTORE_PREEMPT_IF_NEEDED();
10155	VMMRZCallRing3Enable(pVCpu);
10156
10157	#ifdef VBOX_WITH_STATISTICS
10158	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10159	AssertRCReturn(rc, rc);
10160	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10161	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10162	else
10163	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10164	#endif
10165	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10166	return VINF_SUCCESS;
10167	}
10168
10169	/*
10170	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10171	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10172	*/
10173	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10174	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10175	AssertRCReturn(rc, rc);
10176	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10177
10178	PVM pVM = pVCpu->CTX_SUFF(pVM);
10179	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10180	{
10181	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10182	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10183	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10184	if (RT_SUCCESS(rc))
10185	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10186	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10187	}
10188	else
10189	{
10190	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10191	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10192	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10193	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10194	}
10195
10196	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10197	if (RT_SUCCESS(rc))
10198	{
10199	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10200	AssertRCReturn(rc2, rc2);
10201	}
10202	return rc;
10203	}
10204
10205
10206	/**
10207	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10208	* Conditional VM-exit.
10209	*/
10210	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10211	{
10212	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10213	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10214
10215	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10216	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10217	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10218	return VINF_SUCCESS;
10219	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10220	return rc;
10221
10222	RTGCPHYS GCPhys = 0;
10223	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10224
10225	#if 0
10226	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10227	#else
10228	/* Aggressive state sync. for now. */
10229	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10230	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10231	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10232	#endif
10233	AssertRCReturn(rc, rc);
10234
10235	/*
10236	* If we succeed, resume guest execution.
10237	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10238	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10239	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10240	* weird case. See @bugref{6043}.
10241	*/
10242	PVM pVM = pVCpu->CTX_SUFF(pVM);
10243	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10244	rc = VBOXSTRICTRC_VAL(rc2);
10245	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10246	if ( rc == VINF_SUCCESS
10247	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10248	\|\| rc == VERR_PAGE_NOT_PRESENT)
10249	{
10250	/* Successfully handled MMIO operation. */
10251	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10252	\| HM_CHANGED_GUEST_RSP
10253	\| HM_CHANGED_GUEST_RFLAGS
10254	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10255	rc = VINF_SUCCESS;
10256	}
10257	return rc;
10258	}
10259
10260
10261	/**
10262	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10263	* VM-exit.
10264	*/
10265	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10266	{
10267	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10268	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10269
10270	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10271	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10272	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10273	return VINF_SUCCESS;
10274	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10275	return rc;
10276
10277	RTGCPHYS GCPhys = 0;
10278	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10279	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10280	#if 0
10281	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10282	#else
10283	/* Aggressive state sync. for now. */
10284	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10285	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10286	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10287	#endif
10288	AssertRCReturn(rc, rc);
10289
10290	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10291	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10292
10293	RTGCUINT uErrorCode = 0;
10294	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10295	uErrorCode \|= X86_TRAP_PF_ID;
10296	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10297	uErrorCode \|= X86_TRAP_PF_RW;
10298	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10299	uErrorCode \|= X86_TRAP_PF_P;
10300
10301	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10302
10303	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10304	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10305
10306	/* Handle the pagefault trap for the nested shadow table. */
10307	PVM pVM = pVCpu->CTX_SUFF(pVM);
10308	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10309	TRPMResetTrap(pVCpu);
10310
10311	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10312	if ( rc == VINF_SUCCESS
10313	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10314	\|\| rc == VERR_PAGE_NOT_PRESENT)
10315	{
10316	/* Successfully synced our nested page tables. */
10317	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10318	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10319	\| HM_CHANGED_GUEST_RSP
10320	\| HM_CHANGED_GUEST_RFLAGS);
10321	return VINF_SUCCESS;
10322	}
10323
10324	Log4(("EPT return to ring-3 rc=%d\n"));
10325	return rc;
10326	}
10327
10328	/** @} */
10329
10330	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10331	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10332	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10333
10334	/** @name VM-exit exception handlers.
10335	* @{
10336	*/
10337
10338	/**
10339	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10340	*/
10341	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10342	{
10343	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10344	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10345
10346	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10347	AssertRCReturn(rc, rc);
10348
10349	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10350	{
10351	/* Old-style FPU error reporting needs some extra work. */
10352	/** @todo don't fall back to the recompiler, but do it manually. */
10353	return VERR_EM_INTERPRETER;
10354	}
10355
10356	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10357	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10358	return rc;
10359	}
10360
10361
10362	/**
10363	* VM-exit exception handler for #BP (Breakpoint exception).
10364	*/
10365	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10366	{
10367	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10368	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10369
10370	/** @todo Try optimize this by not saving the entire guest state unless
10371	* really needed. */
10372	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10373	AssertRCReturn(rc, rc);
10374
10375	PVM pVM = pVCpu->CTX_SUFF(pVM);
10376	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10377	if (rc == VINF_EM_RAW_GUEST_TRAP)
10378	{
10379	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10380	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10381	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10382	AssertRCReturn(rc, rc);
10383
10384	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10385	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10386	}
10387
10388	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10389	return rc;
10390	}
10391
10392
10393	/**
10394	* VM-exit exception handler for #DB (Debug exception).
10395	*/
10396	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10397	{
10398	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10399	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10400	Log6(("XcptDB\n"));
10401
10402	/*
10403	* Get the DR6-like values from the exit qualification and pass it to DBGF
10404	* for processing.
10405	*/
10406	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10407	AssertRCReturn(rc, rc);
10408
10409	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10410	uint64_t uDR6 = X86_DR6_INIT_VAL;
10411	uDR6 \|= ( pVmxTransient->uExitQualification
10412	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10413
10414	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10415	if (rc == VINF_EM_RAW_GUEST_TRAP)
10416	{
10417	/*
10418	* The exception was for the guest. Update DR6, DR7.GD and
10419	* IA32_DEBUGCTL.LBR before forwarding it.
10420	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10421	*/
10422	VMMRZCallRing3Disable(pVCpu);
10423	HM_DISABLE_PREEMPT_IF_NEEDED();
10424
10425	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10426	pMixedCtx->dr[6] \|= uDR6;
10427	if (CPUMIsGuestDebugStateActive(pVCpu))
10428	ASMSetDR6(pMixedCtx->dr[6]);
10429
10430	HM_RESTORE_PREEMPT_IF_NEEDED();
10431	VMMRZCallRing3Enable(pVCpu);
10432
10433	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10434	AssertRCReturn(rc, rc);
10435
10436	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10437	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10438
10439	/* Paranoia. */
10440	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10441	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10442
10443	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10444	AssertRCReturn(rc, rc);
10445
10446	/*
10447	* Raise #DB in the guest.
10448	*/
10449	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10450	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10451	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10452	AssertRCReturn(rc, rc);
10453	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10454	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10455	return VINF_SUCCESS;
10456	}
10457
10458	/*
10459	* Not a guest trap, must be a hypervisor related debug event then.
10460	* Update DR6 in case someone is interested in it.
10461	*/
10462	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10463	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10464	CPUMSetHyperDR6(pVCpu, uDR6);
10465
10466	return rc;
10467	}
10468
10469
10470	/**
10471	* VM-exit exception handler for #NM (Device-not-available exception: floating
10472	* point exception).
10473	*/
10474	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10475	{
10476	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10477
10478	/* We require CR0 and EFER. EFER is always up-to-date. */
10479	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10480	AssertRCReturn(rc, rc);
10481
10482	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10483	VMMRZCallRing3Disable(pVCpu);
10484	HM_DISABLE_PREEMPT_IF_NEEDED();
10485
10486	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10487	if (pVmxTransient->fWasGuestFPUStateActive)
10488	{
10489	rc = VINF_EM_RAW_GUEST_TRAP;
10490	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10491	}
10492	else
10493	{
10494	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10495	Assert(!pVmxTransient->fWasGuestFPUStateActive);
10496	#endif
10497	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10498	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10499	}
10500
10501	HM_RESTORE_PREEMPT_IF_NEEDED();
10502	VMMRZCallRing3Enable(pVCpu);
10503
10504	if (rc == VINF_SUCCESS)
10505	{
10506	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
10507	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10508	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10509	}
10510	else
10511	{
10512	/* Forward #NM to the guest. */
10513	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10514	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10515	AssertRCReturn(rc, rc);
10516	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10517	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
10518	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10519	}
10520
10521	return VINF_SUCCESS;
10522	}
10523
10524
10525	/**
10526	* VM-exit exception handler for #GP (General-protection exception).
10527	*
10528	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
10529	*/
10530	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10531	{
10532	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10533	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10534
10535	int rc = VERR_INTERNAL_ERROR_5;
10536	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10537	{
10538	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10539	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10540	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10541	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10542	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10543	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10544	AssertRCReturn(rc, rc);
10545	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
10546	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10547	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10548	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10549	return rc;
10550	#else
10551	/* We don't intercept #GP. */
10552	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10553	return VERR_VMX_UNEXPECTED_EXCEPTION;
10554	#endif
10555	}
10556
10557	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10558	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10559
10560	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10561	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10562	AssertRCReturn(rc, rc);
10563
10564	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10565	uint32_t cbOp = 0;
10566	PVM pVM = pVCpu->CTX_SUFF(pVM);
10567	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10568	if (RT_SUCCESS(rc))
10569	{
10570	rc = VINF_SUCCESS;
10571	Assert(cbOp == pDis->cbInstr);
10572	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10573	switch (pDis->pCurInstr->uOpcode)
10574	{
10575	case OP_CLI:
10576	{
10577	pMixedCtx->eflags.Bits.u1IF = 0;
10578	pMixedCtx->rip += pDis->cbInstr;
10579	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10580	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10581	break;
10582	}
10583
10584	case OP_STI:
10585	{
10586	pMixedCtx->eflags.Bits.u1IF = 1;
10587	pMixedCtx->rip += pDis->cbInstr;
10588	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10589	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10590	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10591	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10592	break;
10593	}
10594
10595	case OP_HLT:
10596	{
10597	rc = VINF_EM_HALT;
10598	pMixedCtx->rip += pDis->cbInstr;
10599	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10600	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10601	break;
10602	}
10603
10604	case OP_POPF:
10605	{
10606	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10607	uint32_t cbParm = 0;
10608	uint32_t uMask = 0;
10609	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10610	{
10611	cbParm = 4;
10612	uMask = 0xffffffff;
10613	}
10614	else
10615	{
10616	cbParm = 2;
10617	uMask = 0xffff;
10618	}
10619
10620	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
10621	RTGCPTR GCPtrStack = 0;
10622	X86EFLAGS Eflags;
10623	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10624	&GCPtrStack);
10625	if (RT_SUCCESS(rc))
10626	{
10627	Assert(sizeof(Eflags.u32) >= cbParm);
10628	Eflags.u32 = 0;
10629	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10630	}
10631	if (RT_FAILURE(rc))
10632	{
10633	rc = VERR_EM_INTERPRETER;
10634	break;
10635	}
10636	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10637	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10638	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10639	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
10640	pMixedCtx->esp += cbParm;
10641	pMixedCtx->esp &= uMask;
10642	pMixedCtx->rip += pDis->cbInstr;
10643
10644	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10645	\| HM_CHANGED_GUEST_RSP
10646	\| HM_CHANGED_GUEST_RFLAGS);
10647	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10648	break;
10649	}
10650
10651	case OP_PUSHF:
10652	{
10653	uint32_t cbParm = 0;
10654	uint32_t uMask = 0;
10655	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10656	{
10657	cbParm = 4;
10658	uMask = 0xffffffff;
10659	}
10660	else
10661	{
10662	cbParm = 2;
10663	uMask = 0xffff;
10664	}
10665
10666	/* Get the stack pointer & push the contents of eflags onto the stack. */
10667	RTGCPTR GCPtrStack = 0;
10668	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10669	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10670	if (RT_FAILURE(rc))
10671	{
10672	rc = VERR_EM_INTERPRETER;
10673	break;
10674	}
10675	X86EFLAGS Eflags = pMixedCtx->eflags;
10676	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10677	Eflags.Bits.u1RF = 0;
10678	Eflags.Bits.u1VM = 0;
10679
10680	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10681	if (RT_FAILURE(rc))
10682	{
10683	rc = VERR_EM_INTERPRETER;
10684	break;
10685	}
10686	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10687	pMixedCtx->esp -= cbParm;
10688	pMixedCtx->esp &= uMask;
10689	pMixedCtx->rip += pDis->cbInstr;
10690	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
10691	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10692	break;
10693	}
10694
10695	case OP_IRET:
10696	{
10697	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10698	* instruction reference. */
10699	RTGCPTR GCPtrStack = 0;
10700	uint32_t uMask = 0xffff;
10701	uint16_t aIretFrame[3];
10702	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
10703	{
10704	rc = VERR_EM_INTERPRETER;
10705	break;
10706	}
10707	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10708	&GCPtrStack);
10709	if (RT_SUCCESS(rc))
10710	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10711	if (RT_FAILURE(rc))
10712	{
10713	rc = VERR_EM_INTERPRETER;
10714	break;
10715	}
10716	pMixedCtx->eip = 0;
10717	pMixedCtx->ip = aIretFrame[0];
10718	pMixedCtx->cs.Sel = aIretFrame[1];
10719	pMixedCtx->cs.ValidSel = aIretFrame[1];
10720	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10721	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10722	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10723	pMixedCtx->sp += sizeof(aIretFrame);
10724	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10725	\| HM_CHANGED_GUEST_SEGMENT_REGS
10726	\| HM_CHANGED_GUEST_RSP
10727	\| HM_CHANGED_GUEST_RFLAGS);
10728	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10729	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10730	break;
10731	}
10732
10733	case OP_INT:
10734	{
10735	uint16_t uVector = pDis->Param1.uValue & 0xff;
10736	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10737	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10738	break;
10739	}
10740
10741	case OP_INTO:
10742	{
10743	if (pMixedCtx->eflags.Bits.u1OF)
10744	{
10745	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10746	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10747	}
10748	break;
10749	}
10750
10751	default:
10752	{
10753	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10754	EMCODETYPE_SUPERVISOR);
10755	rc = VBOXSTRICTRC_VAL(rc2);
10756	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
10757	Log4(("#GP rc=%Rrc\n", rc));
10758	break;
10759	}
10760	}
10761	}
10762	else
10763	rc = VERR_EM_INTERPRETER;
10764
10765	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
10766	("#GP Unexpected rc=%Rrc\n", rc));
10767	return rc;
10768	}
10769
10770
10771	/**
10772	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10773	* the exception reported in the VMX transient structure back into the VM.
10774	*
10775	* @remarks Requires uExitIntInfo in the VMX transient structure to be
10776	* up-to-date.
10777	*/
10778	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10779	{
10780	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10781
10782	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10783	hmR0VmxCheckExitDueToEventDelivery(). */
10784	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10785	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10786	AssertRCReturn(rc, rc);
10787	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10788
10789	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10790	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10791	return VINF_SUCCESS;
10792	}
10793
10794
10795	/**
10796	* VM-exit exception handler for #PF (Page-fault exception).
10797	*/
10798	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10799	{
10800	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10801	PVM pVM = pVCpu->CTX_SUFF(pVM);
10802	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10803	rc \|= hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10804	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10805	AssertRCReturn(rc, rc);
10806
10807	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
10808	if (pVM->hm.s.fNestedPaging)
10809	{
10810	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10811	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10812	{
10813	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10814	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10815	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
10816	}
10817	else
10818	{
10819	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10820	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10821	Log4(("Pending #DF due to vectoring #PF. NP\n"));
10822	}
10823	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10824	return rc;
10825	}
10826	#else
10827	Assert(!pVM->hm.s.fNestedPaging);
10828	#endif
10829
10830	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10831	AssertRCReturn(rc, rc);
10832
10833	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10834	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
10835
10836	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
10837	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
10838	(RTGCPTR)pVmxTransient->uExitQualification);
10839
10840	Log4(("#PF: rc=%Rrc\n", rc));
10841	if (rc == VINF_SUCCESS)
10842	{
10843	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10844	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
10845	* memory? We don't update the whole state here... */
10846	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10847	\| HM_CHANGED_GUEST_RSP
10848	\| HM_CHANGED_GUEST_RFLAGS
10849	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10850	TRPMResetTrap(pVCpu);
10851	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10852	return rc;
10853	}
10854	else if (rc == VINF_EM_RAW_GUEST_TRAP)
10855	{
10856	if (!pVmxTransient->fVectoringPF)
10857	{
10858	/* It's a guest page fault and needs to be reflected to the guest. */
10859	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10860	TRPMResetTrap(pVCpu);
10861	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10862	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10863	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10864	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10865	}
10866	else
10867	{
10868	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10869	TRPMResetTrap(pVCpu);
10870	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10871	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10872	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10873	}
10874
10875	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10876	return VINF_SUCCESS;
10877	}
10878
10879	TRPMResetTrap(pVCpu);
10880	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10881	return rc;
10882	}
10883
10884	/** @} */
10885

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49207

Download in other formats: