HMVMXR0.cpp@ 50426

Last change on this file since 50426 was 50426, checked in by vboxsync, 11 years ago
HMR0VMX.cpp: Hacked hmR0VmxSaveHostSegmentRegs to call VMXRestoreHostState when necessary so we don't corrupt the host state should HM_CHANGED_HOST_CONTEXT be set and executed without leaving the HM ring-0 context. Also restricted the dont-bother-to-restore-host-Idtr.limit condition to only include Linux and Solaris as this is known to cause trouble on Windows and Darwin already (HMR0Mixed.mac needs tweaking and testing to complete this change.)
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 474.6 KB

Line
1	/* $Id: HMVMXR0.cpp 50426 2014-02-12 13:16:33Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/x86.h>
23	#include <iprt/asm-amd64-x86.h>
24	#include <iprt/thread.h>
25	#include <iprt/string.h>
26
27	#include "HMInternal.h"
28	#include <VBox/vmm/vm.h>
29	#include "HMVMXR0.h"
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#ifdef VBOX_WITH_REM
37	# include <VBox/vmm/rem.h>
38	#endif
39	#ifdef DEBUG_ramshankar
40	# define HMVMX_SAVE_FULL_GUEST_STATE
41	# define HMVMX_SYNC_FULL_GUEST_STATE
42	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44	# define HMVMX_ALWAYS_TRAP_PF
45	# define HMVMX_ALWAYS_SWAP_FPU_STATE
46	# define HMVMX_ALWAYS_FLUSH_TLB
47	#endif
48
49
50	/*******************************************************************************
51	* Defined Constants And Macros *
52	*******************************************************************************/
53	#if defined(RT_ARCH_AMD64)
54	# define HMVMX_IS_64BIT_HOST_MODE() (true)
55	typedef RTHCUINTREG HMVMXHCUINTREG;
56	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57	extern "C" uint32_t g_fVMXIs64bitHost;
58	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	typedef uint64_t HMVMXHCUINTREG;
60	#else
61	# define HMVMX_IS_64BIT_HOST_MODE() (false)
62	typedef RTHCUINTREG HMVMXHCUINTREG;
63	#endif
64
65	/** Use the function table. */
66	#define HMVMX_USE_FUNCTION_TABLE
67
68	/** Determine which tagged-TLB flush handler to use. */
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74	/** @name Updated-guest-state flags.
75	* @{ */
76	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96	\| HMVMX_UPDATED_GUEST_RSP \
97	\| HMVMX_UPDATED_GUEST_RFLAGS \
98	\| HMVMX_UPDATED_GUEST_CR0 \
99	\| HMVMX_UPDATED_GUEST_CR3 \
100	\| HMVMX_UPDATED_GUEST_CR4 \
101	\| HMVMX_UPDATED_GUEST_GDTR \
102	\| HMVMX_UPDATED_GUEST_IDTR \
103	\| HMVMX_UPDATED_GUEST_LDTR \
104	\| HMVMX_UPDATED_GUEST_TR \
105	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106	\| HMVMX_UPDATED_GUEST_DEBUG \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127	/** @} */
128
129	/** @name
130	* States of the VMCS.
131	*
132	* This does not reflect all possible VMCS states but currently only those
133	* needed for maintaining the VMCS consistently even when thread-context hooks
134	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
135	*/
136	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139	/** @} */
140
141	/**
142	* Exception bitmap mask for real-mode guests (real-on-v86).
143	*
144	* We need to intercept all exceptions manually (except #PF). #NM is also
145	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146	* even in real-mode if we have Nested Paging support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*******************************************************************************
197	* Structures and Typedefs *
198	*******************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG uEflags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u6Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringPF;
278	} VMXTRANSIENT;
279	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284	/** Pointer to VMX transient state. */
285	typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288	/**
289	* MSR-bitmap read permissions.
290	*/
291	typedef enum VMXMSREXITREAD
292	{
293	/** Reading this MSR causes a VM-exit. */
294	VMXMSREXIT_INTERCEPT_READ = 0xb,
295	/** Reading this MSR does not cause a VM-exit. */
296	VMXMSREXIT_PASSTHRU_READ
297	} VMXMSREXITREAD;
298	/** Pointer to MSR-bitmap read permissions. */
299	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311	/** Pointer to MSR-bitmap write permissions. */
312	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315	/**
316	* VMX VM-exit handler.
317	*
318	* @returns VBox status code.
319	* @param pVCpu Pointer to the VMCPU.
320	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
321	* out-of-sync. Make sure to update the required
322	* fields before using them.
323	* @param pVmxTransient Pointer to the VMX-transient structure.
324	*/
325	#ifndef HMVMX_USE_FUNCTION_TABLE
326	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327	#else
328	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	/** Pointer to VM-exit handler. */
330	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331	#endif
332
333
334	/*******************************************************************************
335	* Internal Functions *
336	*******************************************************************************/
337	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343	#endif
344	#ifndef HMVMX_USE_FUNCTION_TABLE
345	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346	# define HMVMX_EXIT_DECL static int
347	#else
348	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349	#endif
350
351	/** @name VM-exit handlers.
352	* @{
353	*/
354	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358	static FNVMXEXITHANDLER hmR0VmxExitSipi;
359	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360	static FNVMXEXITHANDLER hmR0VmxExitSmi;
361	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366	static FNVMXEXITHANDLER hmR0VmxExitHlt;
367	static FNVMXEXITHANDLER hmR0VmxExitInvd;
368	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371	static FNVMXEXITHANDLER hmR0VmxExitRsm;
372	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381	static FNVMXEXITHANDLER hmR0VmxExitMwait;
382	static FNVMXEXITHANDLER hmR0VmxExitMtf;
383	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384	static FNVMXEXITHANDLER hmR0VmxExitPause;
385	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398	/** @} */
399
400	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408	#endif
409	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411	/*******************************************************************************
412	* Global Variables *
413	*******************************************************************************/
414	#ifdef HMVMX_USE_FUNCTION_TABLE
415
416	/**
417	* VMX_EXIT dispatch table.
418	*/
419	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420	{
421	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461	/* 40 UNDEFINED */ hmR0VmxExitPause,
462	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481	};
482	#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484	#ifdef VBOX_STRICT
485	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486	{
487	/* 0 */ "(Not Used)",
488	/* 1 */ "VMCALL executed in VMX root operation.",
489	/* 2 */ "VMCLEAR with invalid physical address.",
490	/* 3 */ "VMCLEAR with VMXON pointer.",
491	/* 4 */ "VMLAUNCH with non-clear VMCS.",
492	/* 5 */ "VMRESUME with non-launched VMCS.",
493	/* 6 */ "VMRESUME after VMXOFF",
494	/* 7 */ "VM entry with invalid control fields.",
495	/* 8 */ "VM entry with invalid host state fields.",
496	/* 9 */ "VMPTRLD with invalid physical address.",
497	/* 10 */ "VMPTRLD with VMXON pointer.",
498	/* 11 */ "VMPTRLD with incorrect revision identifier.",
499	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500	/* 13 */ "VMWRITE to read-only VMCS component.",
501	/* 14 */ "(Not Used)",
502	/* 15 */ "VMXON executed in VMX root operation.",
503	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
504	/* 17 */ "VM entry with non-launched executing VMCS.",
505	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506	/* 19 */ "VMCALL with non-clear VMCS.",
507	/* 20 */ "VMCALL with invalid VM-exit control fields.",
508	/* 21 */ "(Not Used)",
509	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511	/* 24 */ "VMCALL with invalid SMM-monitor features.",
512	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513	/* 26 */ "VM entry with events blocked by MOV SS.",
514	/* 27 */ "(Not Used)",
515	/* 28 */ "Invalid operand to INVEPT/INVVPID."
516	};
517	#endif /* VBOX_STRICT */
518
519
520
521	/**
522	* Updates the VM's last error record. If there was a VMX instruction error,
523	* reads the error data from the VMCS and updates VCPU's last error record as
524	* well.
525	*
526	* @param pVM Pointer to the VM.
527	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528	* VERR_VMX_UNABLE_TO_START_VM or
529	* VERR_VMX_INVALID_VMCS_FIELD).
530	* @param rc The error code.
531	*/
532	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533	{
534	AssertPtr(pVM);
535	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
537	{
538	AssertPtrReturnVoid(pVCpu);
539	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540	}
541	pVM->hm.s.lLastError = rc;
542	}
543
544
545	/**
546	* Reads the VM-entry interruption-information field from the VMCS into the VMX
547	* transient structure.
548	*
549	* @returns VBox status code.
550	* @param pVmxTransient Pointer to the VMX transient structure.
551	*
552	* @remarks No-long-jump zone!!!
553	*/
554	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555	{
556	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557	AssertRCReturn(rc, rc);
558	return VINF_SUCCESS;
559	}
560
561
562	/**
563	* Reads the VM-entry exception error code field from the VMCS into
564	* the VMX transient structure.
565	*
566	* @returns VBox status code.
567	* @param pVmxTransient Pointer to the VMX transient structure.
568	*
569	* @remarks No-long-jump zone!!!
570	*/
571	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572	{
573	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574	AssertRCReturn(rc, rc);
575	return VINF_SUCCESS;
576	}
577
578
579	/**
580	* Reads the VM-entry exception error code field from the VMCS into
581	* the VMX transient structure.
582	*
583	* @returns VBox status code.
584	* @param pVmxTransient Pointer to the VMX transient structure.
585	*
586	* @remarks No-long-jump zone!!!
587	*/
588	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589	{
590	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591	AssertRCReturn(rc, rc);
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the VM-exit interruption-information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*/
603	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the VM-exit interruption error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Reads the VM-exit instruction length field from the VMCS into the VMX
636	* transient structure.
637	*
638	* @returns VBox status code.
639	* @param pVCpu Pointer to the VMCPU.
640	* @param pVmxTransient Pointer to the VMX transient structure.
641	*/
642	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643	{
644	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645	{
646	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647	AssertRCReturn(rc, rc);
648	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649	}
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Reads the VM-exit instruction-information field from the VMCS into
656	* the VMX transient structure.
657	*
658	* @returns VBox status code.
659	* @param pVmxTransient Pointer to the VMX transient structure.
660	*/
661	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662	{
663	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664	{
665	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666	AssertRCReturn(rc, rc);
667	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668	}
669	return VINF_SUCCESS;
670	}
671
672
673	/**
674	* Reads the exit qualification from the VMCS into the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678	* case).
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684	{
685	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the IDT-vectoring information field from the VMCS into the VMX
695	* transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*
700	* @remarks No-long-jump zone!!!
701	*/
702	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703	{
704	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705	{
706	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707	AssertRCReturn(rc, rc);
708	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709	}
710	return VINF_SUCCESS;
711	}
712
713
714	/**
715	* Reads the IDT-vectoring error code from the VMCS into the VMX
716	* transient structure.
717	*
718	* @returns VBox status code.
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724	{
725	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Enters VMX root mode operation on the current CPU.
735	*
736	* @returns VBox status code.
737	* @param pVM Pointer to the VM (optional, can be NULL, after
738	* a resume).
739	* @param HCPhysCpuPage Physical address of the VMXON region.
740	* @param pvCpuPage Pointer to the VMXON region.
741	*/
742	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743	{
744	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
745	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
746	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
747
748	if (pVM)
749	{
750	/* Write the VMCS revision dword to the VMXON region. */
751	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
752	}
753
754	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
755	RTCCUINTREG uEflags = ASMIntDisableFlags();
756
757	/* Enable the VMX bit in CR4 if necessary. */
758	RTCCUINTREG uCr4 = ASMGetCR4();
759	if (!(uCr4 & X86_CR4_VMXE))
760	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
761
762	/* Enter VMX root mode. */
763	int rc = VMXEnable(HCPhysCpuPage);
764	if (RT_FAILURE(rc))
765	ASMSetCR4(uCr4);
766
767	/* Restore interrupts. */
768	ASMSetFlags(uEflags);
769	return rc;
770	}
771
772
773	/**
774	* Exits VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	*/
778	static int hmR0VmxLeaveRootMode(void)
779	{
780	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
781
782	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
783	RTCCUINTREG uEflags = ASMIntDisableFlags();
784
785	/* If we're for some reason not in VMX root mode, then don't leave it. */
786	RTCCUINTREG uHostCR4 = ASMGetCR4();
787
788	int rc;
789	if (uHostCR4 & X86_CR4_VMXE)
790	{
791	/* Exit VMX root mode and clear the VMX bit in CR4. */
792	VMXDisable();
793	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
794	rc = VINF_SUCCESS;
795	}
796	else
797	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
798
799	/* Restore interrupts. */
800	ASMSetFlags(uEflags);
801	return rc;
802	}
803
804
805	/**
806	* Allocates and maps one physically contiguous page. The allocated page is
807	* zero'd out. (Used by various VT-x structures).
808	*
809	* @returns IPRT status code.
810	* @param pMemObj Pointer to the ring-0 memory object.
811	* @param ppVirt Where to store the virtual address of the
812	* allocation.
813	* @param pPhys Where to store the physical address of the
814	* allocation.
815	*/
816	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
817	{
818	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
819	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
820	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
821
822	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
823	if (RT_FAILURE(rc))
824	return rc;
825	ppVirt = RTR0MemObjAddress(pMemObj);
826	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
827	ASMMemZero32(*ppVirt, PAGE_SIZE);
828	return VINF_SUCCESS;
829	}
830
831
832	/**
833	* Frees and unmaps an allocated physical page.
834	*
835	* @param pMemObj Pointer to the ring-0 memory object.
836	* @param ppVirt Where to re-initialize the virtual address of
837	* allocation as 0.
838	* @param pHCPhys Where to re-initialize the physical address of the
839	* allocation as 0.
840	*/
841	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
842	{
843	AssertPtr(pMemObj);
844	AssertPtr(ppVirt);
845	AssertPtr(pHCPhys);
846	if (*pMemObj != NIL_RTR0MEMOBJ)
847	{
848	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
849	AssertRC(rc);
850	*pMemObj = NIL_RTR0MEMOBJ;
851	*ppVirt = 0;
852	*pHCPhys = 0;
853	}
854	}
855
856
857	/**
858	* Worker function to free VT-x related structures.
859	*
860	* @returns IPRT status code.
861	* @param pVM Pointer to the VM.
862	*/
863	static void hmR0VmxStructsFree(PVM pVM)
864	{
865	for (VMCPUID i = 0; i < pVM->cCpus; i++)
866	{
867	PVMCPU pVCpu = &pVM->aCpus[i];
868	AssertPtr(pVCpu);
869
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872
873	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
874	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
875
876	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
878	}
879
880	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
881	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
882	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
883	#endif
884	}
885
886
887	/**
888	* Worker function to allocate VT-x related VM structures.
889	*
890	* @returns IPRT status code.
891	* @param pVM Pointer to the VM.
892	*/
893	static int hmR0VmxStructsAlloc(PVM pVM)
894	{
895	/*
896	* Initialize members up-front so we can cleanup properly on allocation failure.
897	*/
898	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
899	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
900	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
901	pVM->hm.s.vmx.HCPhys##a_Name = 0;
902
903	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
904	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
905	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
906	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
907
908	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
909	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
910	#endif
911	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
912
913	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
914	for (VMCPUID i = 0; i < pVM->cCpus; i++)
915	{
916	PVMCPU pVCpu = &pVM->aCpus[i];
917	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
922	}
923	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
924	#undef VMXLOCAL_INIT_VM_MEMOBJ
925
926	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
927	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
928	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
929	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
930
931	/*
932	* Allocate all the VT-x structures.
933	*/
934	int rc = VINF_SUCCESS;
935	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
936	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
937	if (RT_FAILURE(rc))
938	goto cleanup;
939	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
940	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
941	#endif
942
943	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
944	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
945	{
946	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
947	&pVM->hm.s.vmx.HCPhysApicAccess);
948	if (RT_FAILURE(rc))
949	goto cleanup;
950	}
951
952	/*
953	* Initialize per-VCPU VT-x structures.
954	*/
955	for (VMCPUID i = 0; i < pVM->cCpus; i++)
956	{
957	PVMCPU pVCpu = &pVM->aCpus[i];
958	AssertPtr(pVCpu);
959
960	/* Allocate the VM control structure (VMCS). */
961	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
962	if (RT_FAILURE(rc))
963	goto cleanup;
964
965	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
966	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
967	{
968	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
969	&pVCpu->hm.s.vmx.HCPhysVirtApic);
970	if (RT_FAILURE(rc))
971	goto cleanup;
972	}
973
974	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
975	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
976	{
977	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
978	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
979	if (RT_FAILURE(rc))
980	goto cleanup;
981	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
982	}
983
984	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
985	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
986	if (RT_FAILURE(rc))
987	goto cleanup;
988
989	/* Allocate the VM-exit MSR-load page for the host MSRs. */
990	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	return VINF_SUCCESS;
996
997	cleanup:
998	hmR0VmxStructsFree(pVM);
999	return rc;
1000	}
1001
1002
1003	/**
1004	* Does global VT-x initialization (called during module initialization).
1005	*
1006	* @returns VBox status code.
1007	*/
1008	VMMR0DECL(int) VMXR0GlobalInit(void)
1009	{
1010	#ifdef HMVMX_USE_FUNCTION_TABLE
1011	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1012	# ifdef VBOX_STRICT
1013	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1014	Assert(g_apfnVMExitHandlers[i]);
1015	# endif
1016	#endif
1017	return VINF_SUCCESS;
1018	}
1019
1020
1021	/**
1022	* Does global VT-x termination (called during module termination).
1023	*/
1024	VMMR0DECL(void) VMXR0GlobalTerm()
1025	{
1026	/* Nothing to do currently. */
1027	}
1028
1029
1030	/**
1031	* Sets up and activates VT-x on the current CPU.
1032	*
1033	* @returns VBox status code.
1034	* @param pCpu Pointer to the global CPU info struct.
1035	* @param pVM Pointer to the VM (can be NULL after a host resume
1036	* operation).
1037	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1038	* fEnabledByHost is true).
1039	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1040	* @a fEnabledByHost is true).
1041	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1042	* enable VT-x on the host.
1043	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1044	*/
1045	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1046	void *pvMsrs)
1047	{
1048	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1049	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051
1052	/* Enable VT-x if it's not already enabled by the host. */
1053	if (!fEnabledByHost)
1054	{
1055	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1056	if (RT_FAILURE(rc))
1057	return rc;
1058	}
1059
1060	/*
1061	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1062	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1063	*/
1064	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1065	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1066	{
1067	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1068	pCpu->fFlushAsidBeforeUse = false;
1069	}
1070	else
1071	pCpu->fFlushAsidBeforeUse = true;
1072
1073	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1074	++pCpu->cTlbFlushes;
1075
1076	return VINF_SUCCESS;
1077	}
1078
1079
1080	/**
1081	* Deactivates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pCpu Pointer to the global CPU info struct.
1085	* @param pvCpuPage Pointer to the VMXON region.
1086	* @param HCPhysCpuPage Physical address of the VMXON region.
1087	*
1088	* @remarks This function should never be called when SUPR0EnableVTx() or
1089	* similar was used to enable VT-x on the host.
1090	*/
1091	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1092	{
1093	NOREF(pCpu);
1094	NOREF(pvCpuPage);
1095	NOREF(HCPhysCpuPage);
1096
1097	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1098	return hmR0VmxLeaveRootMode();
1099	}
1100
1101
1102	/**
1103	* Sets the permission bits for the specified MSR in the MSR bitmap.
1104	*
1105	* @param pVCpu Pointer to the VMCPU.
1106	* @param uMSR The MSR value.
1107	* @param enmRead Whether reading this MSR causes a VM-exit.
1108	* @param enmWrite Whether writing this MSR causes a VM-exit.
1109	*/
1110	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1111	{
1112	int32_t iBit;
1113	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1114
1115	/*
1116	* Layout:
1117	* 0x000 - 0x3ff - Low MSR read bits
1118	* 0x400 - 0x7ff - High MSR read bits
1119	* 0x800 - 0xbff - Low MSR write bits
1120	* 0xc00 - 0xfff - High MSR write bits
1121	*/
1122	if (uMsr <= 0x00001FFF)
1123	iBit = uMsr;
1124	else if ( uMsr >= 0xC0000000
1125	&& uMsr <= 0xC0001FFF)
1126	{
1127	iBit = (uMsr - 0xC0000000);
1128	pbMsrBitmap += 0x400;
1129	}
1130	else
1131	{
1132	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1133	return;
1134	}
1135
1136	Assert(iBit <= 0x1fff);
1137	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1138	ASMBitSet(pbMsrBitmap, iBit);
1139	else
1140	ASMBitClear(pbMsrBitmap, iBit);
1141
1142	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1143	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1144	else
1145	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1146	}
1147
1148
1149	#ifdef VBOX_STRICT
1150	/**
1151	* Gets the permission bits for the specified MSR in the MSR bitmap.
1152	*
1153	* @returns VBox status code.
1154	* @retval VINF_SUCCESS if the specified MSR is found.
1155	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1156	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1157	*
1158	* @param pVCpu Pointer to the VMCPU.
1159	* @param uMsr The MSR.
1160	* @param penmRead Where to store the read permissions.
1161	* @param penmWrite Where to store the write permissions.
1162	*/
1163	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1164	{
1165	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1166	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1167	int32_t iBit;
1168	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170	/* See hmR0VmxSetMsrPermission() for the layout. */
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if ( uMsr >= 0xC0000000
1174	&& uMsr <= 0xC0001FFF)
1175	{
1176	iBit = (uMsr - 0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	{
1181	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182	return VERR_NOT_SUPPORTED;
1183	}
1184
1185	Assert(iBit <= 0x1fff);
1186	if (ASMBitTest(pbMsrBitmap, iBit))
1187	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1188	else
1189	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1190
1191	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1192	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1193	else
1194	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1195	return VINF_SUCCESS;
1196	}
1197	#endif /* VBOX_STRICT */
1198
1199
1200	/**
1201	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1202	* area.
1203	*
1204	* @returns VBox status code.
1205	* @param pVCpu Pointer to the VMCPU.
1206	* @param cMsrs The number of MSRs.
1207	*/
1208	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1209	{
1210	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1211	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1212	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1213	{
1214	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1215	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1216	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1217	}
1218
1219	/* Update number of guest MSRs to load/store across the world-switch. */
1220	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1221	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222
1223	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1224	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1225
1226	/* Update the VCPU's copy of the MSR count. */
1227	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1228
1229	return VINF_SUCCESS;
1230	}
1231
1232
1233	/**
1234	* Adds a new (or updates the value of an existing) guest/host MSR
1235	* pair to be swapped during the world-switch as part of the
1236	* auto-load/store MSR area in the VMCS.
1237	*
1238	* @returns VBox status code.
1239	* @param pVCpu Pointer to the VMCPU.
1240	* @param uMsr The MSR.
1241	* @param uGuestMsr Value of the guest MSR.
1242	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1243	* necessary.
1244	*/
1245	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1246	{
1247	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1248	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1249	uint32_t i;
1250	for (i = 0; i < cMsrs; i++)
1251	{
1252	if (pGuestMsr->u32Msr == uMsr)
1253	break;
1254	pGuestMsr++;
1255	}
1256
1257	bool fAdded = false;
1258	if (i == cMsrs)
1259	{
1260	++cMsrs;
1261	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1262	AssertRCReturn(rc, rc);
1263
1264	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1265	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1266	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1267
1268	fAdded = true;
1269	}
1270
1271	/* Update the MSR values in the auto-load/store MSR area. */
1272	pGuestMsr->u32Msr = uMsr;
1273	pGuestMsr->u64Value = uGuestMsrValue;
1274
1275	/* Create/update the MSR slot in the host MSR area. */
1276	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1277	pHostMsr += i;
1278	pHostMsr->u32Msr = uMsr;
1279
1280	/*
1281	* Update the host MSR only when requested by the caller AND when we're
1282	* adding it to the auto-load/store area. Otherwise, it would have been
1283	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1284	*/
1285	if ( fAdded
1286	&& fUpdateHostMsr)
1287	{
1288	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1289	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1290	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1291	}
1292
1293	return VINF_SUCCESS;
1294	}
1295
1296
1297	/**
1298	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1299	* auto-load/store MSR area in the VMCS.
1300	*
1301	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1302	* area.
1303	*
1304	* @returns VBox status code.
1305	* @param pVCpu Pointer to the VMCPU.
1306	* @param uMsr The MSR.
1307	*/
1308	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1309	{
1310	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1311	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1312	for (uint32_t i = 0; i < cMsrs; i++)
1313	{
1314	/* Find the MSR. */
1315	if (pGuestMsr->u32Msr == uMsr)
1316	{
1317	/* If it's the last MSR, simply reduce the count. */
1318	if (i == cMsrs - 1)
1319	{
1320	--cMsrs;
1321	break;
1322	}
1323
1324	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1325	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1326	pLastGuestMsr += cMsrs;
1327	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1328	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1329
1330	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1331	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1332	pLastHostMsr += cMsrs;
1333	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1334	pHostMsr->u64Value = pLastHostMsr->u64Value;
1335	--cMsrs;
1336	break;
1337	}
1338	pGuestMsr++;
1339	}
1340
1341	/* Update the VMCS if the count changed (meaning the MSR was found). */
1342	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1343	{
1344	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1345	AssertRCReturn(rc, rc);
1346
1347	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1348	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1349	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1350	}
1351
1352	return VINF_SUCCESS;
1353	}
1354
1355
1356	/**
1357	* Checks if the specified guest MSR is part of the auto-load/store area in
1358	* the VMCS.
1359	*
1360	* @returns true if found, false otherwise.
1361	* @param pVCpu Pointer to the VMCPU.
1362	* @param uMsr The MSR to find.
1363	*/
1364	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1365	{
1366	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1367	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1368
1369	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1370	{
1371	if (pGuestMsr->u32Msr == uMsr)
1372	return true;
1373	}
1374	return false;
1375	}
1376
1377
1378	/**
1379	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1380	*
1381	* @param pVCpu Pointer to the VMCPU.
1382	*
1383	* @remarks No-long-jump zone!!!
1384	*/
1385	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1386	{
1387	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1388	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1389	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1390	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1391
1392	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1393	{
1394	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1395	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1396	}
1397
1398	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1399	}
1400
1401
1402	#if HC_ARCH_BITS == 64
1403	/**
1404	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1405	* perform lazy restoration of the host MSRs while leaving VT-x.
1406	*
1407	* @param pVCpu Pointer to the VMCPU.
1408	*
1409	* @remarks No-long-jump zone!!!
1410	*/
1411	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1412	{
1413	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1414
1415	/*
1416	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1417	*/
1418	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1419	{
1420	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1421	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1422	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1423	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1424	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1425	}
1426	}
1427
1428
1429	/**
1430	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1431	* lazily while leaving VT-x.
1432	*
1433	* @returns true if it does, false otherwise.
1434	* @param pVCpu Pointer to the VMCPU.
1435	* @param uMsr The MSR to check.
1436	*/
1437	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1438	{
1439	NOREF(pVCpu);
1440	switch (uMsr)
1441	{
1442	case MSR_K8_LSTAR:
1443	case MSR_K6_STAR:
1444	case MSR_K8_SF_MASK:
1445	case MSR_K8_KERNEL_GS_BASE:
1446	return true;
1447	}
1448	return false;
1449	}
1450
1451
1452	/**
1453	* Saves a set of guests MSRs back into the guest-CPU context.
1454	*
1455	* @param pVCpu Pointer to the VMCPU.
1456	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1457	* out-of-sync. Make sure to update the required fields
1458	* before using them.
1459	*
1460	* @remarks No-long-jump zone!!!
1461	*/
1462	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1463	{
1464	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1465	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1466
1467	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1468	{
1469	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1470	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1471	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1472	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1473	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1474	}
1475	}
1476
1477
1478	/**
1479	* Loads a set of guests MSRs to allow read/passthru to the guest.
1480	*
1481	* The name of this function is slightly confusing. This function does NOT
1482	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1483	* common prefix for functions dealing with "lazy restoration" of the shared
1484	* MSRs.
1485	*
1486	* @param pVCpu Pointer to the VMCPU.
1487	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1488	* out-of-sync. Make sure to update the required fields
1489	* before using them.
1490	*
1491	* @remarks No-long-jump zone!!!
1492	*/
1493	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1494	{
1495	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1497
1498	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1499	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1500	{
1501	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1502	do { \
1503	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1504	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1505	else \
1506	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1507	} while (0)
1508
1509	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1510	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1511	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1512	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1513	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1514	}
1515	else
1516	{
1517	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1518	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1519	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1520	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1521	}
1522	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1523	}
1524
1525
1526	/**
1527	* Performs lazy restoration of the set of host MSRs if they were previously
1528	* loaded with guest MSR values.
1529	*
1530	* @param pVCpu Pointer to the VMCPU.
1531	*
1532	* @remarks No-long-jump zone!!!
1533	* @remarks The guest MSRs should have been saved back into the guest-CPU
1534	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1535	*/
1536	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1537	{
1538	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1539	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1540
1541	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1542	{
1543	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1544	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1545	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1546	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1547	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1548	}
1549	pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST \| VMX_RESTORE_HOST_MSR_SAVED_HOST);
1550	}
1551	#endif /* HC_ARCH_BITS == 64 */
1552
1553
1554	#ifdef VBOX_STRICT
1555	/**
1556	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1557	* VMCS are correct.
1558	*
1559	* @param pVCpu Pointer to the VMCPU.
1560	*/
1561	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1562	{
1563	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1564
1565	/* Verify MSR counts in the VMCS are what we think it should be. */
1566	uint32_t cMsrs;
1567	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1568	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1569
1570	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1571	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1572
1573	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1574	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1575
1576	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1577	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1578	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1579	{
1580	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1581	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1582	pGuestMsr->u32Msr));
1583
1584	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1585	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1586	pHostMsr->u64Value, u64Msr));
1587
1588	/* Verify that the permissions are as expected in the MSR bitmap. */
1589	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1590	{
1591	VMXMSREXITREAD enmRead;
1592	VMXMSREXITWRITE enmWrite;
1593	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1594	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1595	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1596	pGuestMsr->u32Msr));
1597	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1598	pGuestMsr->u32Msr));
1599	}
1600	}
1601	}
1602	# endif /* VBOX_STRICT */
1603
1604
1605	/**
1606	* Flushes the TLB using EPT.
1607	*
1608	* @returns VBox status code.
1609	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1610	* enmFlush).
1611	* @param enmFlush Type of flush.
1612	*
1613	* @remarks Caller is responsible for making sure this function is called only
1614	* when NestedPaging is supported and providing @a enmFlush that is
1615	* supported by the CPU.
1616	* @remarks Can be called with interrupts disabled.
1617	*/
1618	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1619	{
1620	uint64_t au64Descriptor[2];
1621	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1622	au64Descriptor[0] = 0;
1623	else
1624	{
1625	Assert(pVCpu);
1626	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1627	}
1628	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1629
1630	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1631	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1632	rc));
1633	if ( RT_SUCCESS(rc)
1634	&& pVCpu)
1635	{
1636	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1637	}
1638	}
1639
1640
1641	/**
1642	* Flushes the TLB using VPID.
1643	*
1644	* @returns VBox status code.
1645	* @param pVM Pointer to the VM.
1646	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1647	* enmFlush).
1648	* @param enmFlush Type of flush.
1649	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1650	* on @a enmFlush).
1651	*
1652	* @remarks Can be called with interrupts disabled.
1653	*/
1654	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1655	{
1656	NOREF(pVM);
1657	AssertPtr(pVM);
1658	Assert(pVM->hm.s.vmx.fVpid);
1659
1660	uint64_t au64Descriptor[2];
1661	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1662	{
1663	au64Descriptor[0] = 0;
1664	au64Descriptor[1] = 0;
1665	}
1666	else
1667	{
1668	AssertPtr(pVCpu);
1669	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1670	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1671	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1672	au64Descriptor[1] = GCPtr;
1673	}
1674
1675	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1676	AssertMsg(rc == VINF_SUCCESS,
1677	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1678	if ( RT_SUCCESS(rc)
1679	&& pVCpu)
1680	{
1681	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1682	}
1683	}
1684
1685
1686	/**
1687	* Invalidates a guest page by guest virtual address. Only relevant for
1688	* EPT/VPID, otherwise there is nothing really to invalidate.
1689	*
1690	* @returns VBox status code.
1691	* @param pVM Pointer to the VM.
1692	* @param pVCpu Pointer to the VMCPU.
1693	* @param GCVirt Guest virtual address of the page to invalidate.
1694	*/
1695	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1696	{
1697	AssertPtr(pVM);
1698	AssertPtr(pVCpu);
1699	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1700
1701	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1702	if (!fFlushPending)
1703	{
1704	/*
1705	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1706	* See @bugref{6043} and @bugref{6177}.
1707	*
1708	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1709	* function maybe called in a loop with individual addresses.
1710	*/
1711	if (pVM->hm.s.vmx.fVpid)
1712	{
1713	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1714	{
1715	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1716	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1717	}
1718	else
1719	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1720	}
1721	else if (pVM->hm.s.fNestedPaging)
1722	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1723	}
1724
1725	return VINF_SUCCESS;
1726	}
1727
1728
1729	/**
1730	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1731	* otherwise there is nothing really to invalidate.
1732	*
1733	* @returns VBox status code.
1734	* @param pVM Pointer to the VM.
1735	* @param pVCpu Pointer to the VMCPU.
1736	* @param GCPhys Guest physical address of the page to invalidate.
1737	*/
1738	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1739	{
1740	NOREF(pVM); NOREF(GCPhys);
1741	LogFlowFunc(("%RGp\n", GCPhys));
1742
1743	/*
1744	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1745	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1746	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1747	*/
1748	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1749	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1750	return VINF_SUCCESS;
1751	}
1752
1753
1754	/**
1755	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1756	* case where neither EPT nor VPID is supported by the CPU.
1757	*
1758	* @param pVM Pointer to the VM.
1759	* @param pVCpu Pointer to the VMCPU.
1760	* @param pCpu Pointer to the global HM struct.
1761	*
1762	* @remarks Called with interrupts disabled.
1763	*/
1764	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1765	{
1766	AssertPtr(pVCpu);
1767	AssertPtr(pCpu);
1768	NOREF(pVM);
1769
1770	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1771
1772	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1773	#if 0
1774	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1775	pVCpu->hm.s.TlbShootdown.cPages = 0;
1776	#endif
1777
1778	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1779	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1780	pVCpu->hm.s.fForceTLBFlush = false;
1781	return;
1782	}
1783
1784
1785	/**
1786	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1787	*
1788	* @param pVM Pointer to the VM.
1789	* @param pVCpu Pointer to the VMCPU.
1790	* @param pCpu Pointer to the global HM CPU struct.
1791	* @remarks All references to "ASID" in this function pertains to "VPID" in
1792	* Intel's nomenclature. The reason is, to avoid confusion in compare
1793	* statements since the host-CPU copies are named "ASID".
1794	*
1795	* @remarks Called with interrupts disabled.
1796	*/
1797	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1798	{
1799	#ifdef VBOX_WITH_STATISTICS
1800	bool fTlbFlushed = false;
1801	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1802	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1803	if (!fTlbFlushed) \
1804	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1805	} while (0)
1806	#else
1807	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1808	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1809	#endif
1810
1811	AssertPtr(pVM);
1812	AssertPtr(pCpu);
1813	AssertPtr(pVCpu);
1814	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1815	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1816	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1817
1818
1819	/*
1820	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1821	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1822	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1823	*/
1824	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1825	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1826	{
1827	++pCpu->uCurrentAsid;
1828	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1829	{
1830	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1831	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1832	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1833	}
1834
1835	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1836	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1837	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1838
1839	/*
1840	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1841	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1842	*/
1843	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1844	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1845	HMVMX_SET_TAGGED_TLB_FLUSHED();
1846	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1847	}
1848
1849	/* Check for explicit TLB shootdowns. */
1850	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1851	{
1852	/*
1853	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1854	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1855	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1856	* but not guest-physical mappings.
1857	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1858	*/
1859	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1860	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1861	HMVMX_SET_TAGGED_TLB_FLUSHED();
1862	}
1863
1864	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1865	* where it is commented out. Support individual entry flushing
1866	* someday. */
1867	#if 0
1868	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1869	{
1870	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1871
1872	/*
1873	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1874	* as supported by the CPU.
1875	*/
1876	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1877	{
1878	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1879	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1880	}
1881	else
1882	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1883
1884	HMVMX_SET_TAGGED_TLB_FLUSHED();
1885	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1886	pVCpu->hm.s.TlbShootdown.cPages = 0;
1887	}
1888	#endif
1889
1890	pVCpu->hm.s.fForceTLBFlush = false;
1891
1892	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1893
1894	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1895	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1896	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1897	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1898	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1899	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1900	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1901	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1902
1903	/* Update VMCS with the VPID. */
1904	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1905	AssertRC(rc);
1906
1907	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1908	}
1909
1910
1911	/**
1912	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1913	*
1914	* @returns VBox status code.
1915	* @param pVM Pointer to the VM.
1916	* @param pVCpu Pointer to the VMCPU.
1917	* @param pCpu Pointer to the global HM CPU struct.
1918	*
1919	* @remarks Called with interrupts disabled.
1920	*/
1921	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1922	{
1923	AssertPtr(pVM);
1924	AssertPtr(pVCpu);
1925	AssertPtr(pCpu);
1926	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1927	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1928
1929	/*
1930	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1931	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1932	*/
1933	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1934	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1935	{
1936	pVCpu->hm.s.fForceTLBFlush = true;
1937	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1938	}
1939
1940	/* Check for explicit TLB shootdown flushes. */
1941	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1942	{
1943	pVCpu->hm.s.fForceTLBFlush = true;
1944	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1945	}
1946
1947	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1948	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1949
1950	if (pVCpu->hm.s.fForceTLBFlush)
1951	{
1952	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1953	pVCpu->hm.s.fForceTLBFlush = false;
1954	}
1955	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1956	* where it is commented out. Support individual entry flushing
1957	* someday. */
1958	#if 0
1959	else
1960	{
1961	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1962	{
1963	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1964	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1965	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1966	}
1967	else
1968	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1969
1970	pVCpu->hm.s.TlbShootdown.cPages = 0;
1971	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1972	}
1973	#endif
1974	}
1975
1976
1977	/**
1978	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1979	*
1980	* @returns VBox status code.
1981	* @param pVM Pointer to the VM.
1982	* @param pVCpu Pointer to the VMCPU.
1983	* @param pCpu Pointer to the global HM CPU struct.
1984	*
1985	* @remarks Called with interrupts disabled.
1986	*/
1987	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1988	{
1989	AssertPtr(pVM);
1990	AssertPtr(pVCpu);
1991	AssertPtr(pCpu);
1992	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1993	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1994
1995	/*
1996	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1997	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1998	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1999	*/
2000	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2001	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2002	{
2003	pVCpu->hm.s.fForceTLBFlush = true;
2004	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2005	}
2006
2007	/* Check for explicit TLB shootdown flushes. */
2008	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2009	{
2010	/*
2011	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2012	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2013	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2014	*/
2015	pVCpu->hm.s.fForceTLBFlush = true;
2016	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2017	}
2018
2019	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2020	if (pVCpu->hm.s.fForceTLBFlush)
2021	{
2022	++pCpu->uCurrentAsid;
2023	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2024	{
2025	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2026	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2027	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2028	}
2029
2030	pVCpu->hm.s.fForceTLBFlush = false;
2031	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2032	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2033	if (pCpu->fFlushAsidBeforeUse)
2034	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2035	}
2036	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2037	* where it is commented out. Support individual entry flushing
2038	* someday. */
2039	#if 0
2040	else
2041	{
2042	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2043	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2044	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2045	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2046
2047	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2048	{
2049	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2050	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2051	{
2052	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2053	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2054	}
2055	else
2056	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2057
2058	pVCpu->hm.s.TlbShootdown.cPages = 0;
2059	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2060	}
2061	else
2062	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2063	}
2064	#endif
2065
2066	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2067	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2068	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2069	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
2070	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2071	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2072
2073	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2074	AssertRC(rc);
2075	}
2076
2077
2078	/**
2079	* Flushes the guest TLB entry based on CPU capabilities.
2080	*
2081	* @param pVCpu Pointer to the VMCPU.
2082	* @param pCpu Pointer to the global HM CPU struct.
2083	*/
2084	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2085	{
2086	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2087	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2088	#endif
2089	PVM pVM = pVCpu->CTX_SUFF(pVM);
2090	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2091	{
2092	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2093	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2094	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2095	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2096	default:
2097	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2098	break;
2099	}
2100
2101	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2102	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2103
2104	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
2105	}
2106
2107
2108	/**
2109	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2110	* TLB entries from the host TLB before VM-entry.
2111	*
2112	* @returns VBox status code.
2113	* @param pVM Pointer to the VM.
2114	*/
2115	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2116	{
2117	/*
2118	* Determine optimal flush type for Nested Paging.
2119	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2120	* guest execution (see hmR3InitFinalizeR0()).
2121	*/
2122	if (pVM->hm.s.fNestedPaging)
2123	{
2124	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2125	{
2126	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2127	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2128	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2129	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2130	else
2131	{
2132	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2133	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2134	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2135	}
2136
2137	/* Make sure the write-back cacheable memory type for EPT is supported. */
2138	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2139	{
2140	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2141	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2142	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2143	}
2144	}
2145	else
2146	{
2147	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2148	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2149	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2150	}
2151	}
2152
2153	/*
2154	* Determine optimal flush type for VPID.
2155	*/
2156	if (pVM->hm.s.vmx.fVpid)
2157	{
2158	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2159	{
2160	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2161	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2162	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2163	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2164	else
2165	{
2166	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2167	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2168	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2169	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2170	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2171	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2172	pVM->hm.s.vmx.fVpid = false;
2173	}
2174	}
2175	else
2176	{
2177	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2178	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2179	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2180	pVM->hm.s.vmx.fVpid = false;
2181	}
2182	}
2183
2184	/*
2185	* Setup the handler for flushing tagged-TLBs.
2186	*/
2187	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2188	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2189	else if (pVM->hm.s.fNestedPaging)
2190	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2191	else if (pVM->hm.s.vmx.fVpid)
2192	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2193	else
2194	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2195	return VINF_SUCCESS;
2196	}
2197
2198
2199	/**
2200	* Sets up pin-based VM-execution controls in the VMCS.
2201	*
2202	* @returns VBox status code.
2203	* @param pVM Pointer to the VM.
2204	* @param pVCpu Pointer to the VMCPU.
2205	*/
2206	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2207	{
2208	AssertPtr(pVM);
2209	AssertPtr(pVCpu);
2210
2211	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2212	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2213
2214	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2215	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2216	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2217
2218	/* Enable the VMX preemption timer. */
2219	if (pVM->hm.s.vmx.fUsePreemptTimer)
2220	{
2221	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2222	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2223	}
2224
2225	if ((val & zap) != val)
2226	{
2227	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2228	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2229	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2230	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2231	}
2232
2233	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2234	AssertRCReturn(rc, rc);
2235
2236	/* Update VCPU with the currently set pin-based VM-execution controls. */
2237	pVCpu->hm.s.vmx.u32PinCtls = val;
2238	return rc;
2239	}
2240
2241
2242	/**
2243	* Sets up processor-based VM-execution controls in the VMCS.
2244	*
2245	* @returns VBox status code.
2246	* @param pVM Pointer to the VM.
2247	* @param pVMCPU Pointer to the VMCPU.
2248	*/
2249	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2250	{
2251	AssertPtr(pVM);
2252	AssertPtr(pVCpu);
2253
2254	int rc = VERR_INTERNAL_ERROR_5;
2255	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2256	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2257
2258	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2259	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2260	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2261	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2262	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2263	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2264	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2265
2266	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2267	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2268	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2269	{
2270	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2271	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2272	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2273	}
2274
2275	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2276	if (!pVM->hm.s.fNestedPaging)
2277	{
2278	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2279	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2280	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2281	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2282	}
2283
2284	/* Use TPR shadowing if supported by the CPU. */
2285	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2286	{
2287	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2288	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2289	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2290	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2291	AssertRCReturn(rc, rc);
2292
2293	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2294	/* CR8 writes causes a VM-exit based on TPR threshold. */
2295	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2296	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2297	}
2298	else
2299	{
2300	/*
2301	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2302	* Set this control only for 64-bit guests.
2303	*/
2304	if (pVM->hm.s.fAllow64BitGuests)
2305	{
2306	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2307	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2308	}
2309	}
2310
2311	/* Use MSR-bitmaps if supported by the CPU. */
2312	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2313	{
2314	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2315
2316	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2317	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2318	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2319	AssertRCReturn(rc, rc);
2320
2321	/*
2322	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2323	* automatically as dedicated fields in the VMCS.
2324	*/
2325	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2326	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2327	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2328	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2329	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2330
2331	#if HC_ARCH_BITS == 64
2332	/*
2333	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2334	*/
2335	if (pVM->hm.s.fAllow64BitGuests)
2336	{
2337	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2338	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2339	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2340	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2341	}
2342	#endif
2343	}
2344
2345	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2346	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2347	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2348
2349	if ((val & zap) != val)
2350	{
2351	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2352	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2353	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2354	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2355	}
2356
2357	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2358	AssertRCReturn(rc, rc);
2359
2360	/* Update VCPU with the currently set processor-based VM-execution controls. */
2361	pVCpu->hm.s.vmx.u32ProcCtls = val;
2362
2363	/*
2364	* Secondary processor-based VM-execution controls.
2365	*/
2366	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2367	{
2368	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2372	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2373
2374	if (pVM->hm.s.fNestedPaging)
2375	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2376	else
2377	{
2378	/*
2379	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2380	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2381	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2382	*/
2383	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2384	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2385	}
2386
2387	if (pVM->hm.s.vmx.fVpid)
2388	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2389
2390	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2391	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2392
2393	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2394	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2395	* done dynamically. */
2396	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2397	{
2398	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2399	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2400	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2401	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2402	AssertRCReturn(rc, rc);
2403	}
2404
2405	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2406	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2407
2408	if ((val & zap) != val)
2409	{
2410	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2411	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2412	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2413	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2414	}
2415
2416	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2417	AssertRCReturn(rc, rc);
2418
2419	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2420	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2421	}
2422	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2423	{
2424	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2425	"available\n"));
2426	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2427	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2428	}
2429
2430	return VINF_SUCCESS;
2431	}
2432
2433
2434	/**
2435	* Sets up miscellaneous (everything other than Pin & Processor-based
2436	* VM-execution) control fields in the VMCS.
2437	*
2438	* @returns VBox status code.
2439	* @param pVM Pointer to the VM.
2440	* @param pVCpu Pointer to the VMCPU.
2441	*/
2442	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2443	{
2444	NOREF(pVM);
2445	AssertPtr(pVM);
2446	AssertPtr(pVCpu);
2447
2448	int rc = VERR_GENERAL_FAILURE;
2449
2450	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2451	#if 0
2452	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2453	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2454	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2455
2456	/*
2457	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2458	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2459	* We thus use the exception bitmap to control it rather than use both.
2460	*/
2461	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2462	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2463
2464	/** @todo Explore possibility of using IO-bitmaps. */
2465	/* All IO & IOIO instructions cause VM-exits. */
2466	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2467	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2468
2469	/* Initialize the MSR-bitmap area. */
2470	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2471	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2472	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2473	#endif
2474
2475	/* Setup MSR auto-load/store area. */
2476	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2477	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2478	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2479	AssertRCReturn(rc, rc);
2480	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2481	AssertRCReturn(rc, rc);
2482
2483	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2484	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2485	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2486	AssertRCReturn(rc, rc);
2487
2488	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2489	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2490	AssertRCReturn(rc, rc);
2491
2492	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2493	#if 0
2494	/* Setup debug controls */
2495	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2496	AssertRCReturn(rc, rc);
2497	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2498	AssertRCReturn(rc, rc);
2499	#endif
2500
2501	return rc;
2502	}
2503
2504
2505	/**
2506	* Sets up the initial exception bitmap in the VMCS based on static conditions
2507	* (i.e. conditions that cannot ever change after starting the VM).
2508	*
2509	* @returns VBox status code.
2510	* @param pVM Pointer to the VM.
2511	* @param pVCpu Pointer to the VMCPU.
2512	*/
2513	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2514	{
2515	AssertPtr(pVM);
2516	AssertPtr(pVCpu);
2517
2518	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2519
2520	uint32_t u32XcptBitmap = 0;
2521
2522	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2523	if (!pVM->hm.s.fNestedPaging)
2524	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2525
2526	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2527	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2528	AssertRCReturn(rc, rc);
2529	return rc;
2530	}
2531
2532
2533	/**
2534	* Sets up the initial guest-state mask. The guest-state mask is consulted
2535	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2536	* for the nested virtualization case (as it would cause a VM-exit).
2537	*
2538	* @param pVCpu Pointer to the VMCPU.
2539	*/
2540	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2541	{
2542	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2543	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2544	return VINF_SUCCESS;
2545	}
2546
2547
2548	/**
2549	* Does per-VM VT-x initialization.
2550	*
2551	* @returns VBox status code.
2552	* @param pVM Pointer to the VM.
2553	*/
2554	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2555	{
2556	LogFlowFunc(("pVM=%p\n", pVM));
2557
2558	int rc = hmR0VmxStructsAlloc(pVM);
2559	if (RT_FAILURE(rc))
2560	{
2561	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2562	return rc;
2563	}
2564
2565	return VINF_SUCCESS;
2566	}
2567
2568
2569	/**
2570	* Does per-VM VT-x termination.
2571	*
2572	* @returns VBox status code.
2573	* @param pVM Pointer to the VM.
2574	*/
2575	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2576	{
2577	LogFlowFunc(("pVM=%p\n", pVM));
2578
2579	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2580	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2581	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2582	#endif
2583	hmR0VmxStructsFree(pVM);
2584	return VINF_SUCCESS;
2585	}
2586
2587
2588	/**
2589	* Sets up the VM for execution under VT-x.
2590	* This function is only called once per-VM during initialization.
2591	*
2592	* @returns VBox status code.
2593	* @param pVM Pointer to the VM.
2594	*/
2595	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2596	{
2597	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2598	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2599
2600	LogFlowFunc(("pVM=%p\n", pVM));
2601
2602	/*
2603	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2604	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2605	*/
2606	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2607	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2608	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2609	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2610	{
2611	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2612	return VERR_INTERNAL_ERROR;
2613	}
2614
2615	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2616	/*
2617	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2618	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2619	*/
2620	if ( pVM->hm.s.fAllow64BitGuests
2621	&& !HMVMX_IS_64BIT_HOST_MODE())
2622	{
2623	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2624	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2625	}
2626	#endif
2627
2628	/* Initialize these always, see hmR3InitFinalizeR0().*/
2629	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2630	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2631
2632	/* Setup the tagged-TLB flush handlers. */
2633	int rc = hmR0VmxSetupTaggedTlb(pVM);
2634	if (RT_FAILURE(rc))
2635	{
2636	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2637	return rc;
2638	}
2639
2640	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2641	{
2642	PVMCPU pVCpu = &pVM->aCpus[i];
2643	AssertPtr(pVCpu);
2644	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2645
2646	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2647	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2648
2649	/* Set revision dword at the beginning of the VMCS structure. */
2650	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2651
2652	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2653	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2654	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2655	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2656
2657	/* Load this VMCS as the current VMCS. */
2658	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2659	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2660	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2661
2662	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2663	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2664	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2665
2666	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2667	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2668	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2669
2670	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2671	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2672	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2673
2674	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2675	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2676	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2677
2678	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2679	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2680	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2681
2682	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2683	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2684	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2685	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2686	#endif
2687
2688	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2689	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2690	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2691	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2692
2693	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2694
2695	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2696	}
2697
2698	return VINF_SUCCESS;
2699	}
2700
2701
2702	/**
2703	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2704	* the VMCS.
2705	*
2706	* @returns VBox status code.
2707	* @param pVM Pointer to the VM.
2708	* @param pVCpu Pointer to the VMCPU.
2709	*/
2710	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2711	{
2712	NOREF(pVM); NOREF(pVCpu);
2713
2714	RTCCUINTREG uReg = ASMGetCR0();
2715	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2716	AssertRCReturn(rc, rc);
2717
2718	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2719	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2720	if (HMVMX_IS_64BIT_HOST_MODE())
2721	{
2722	uint64_t uRegCR3 = HMR0Get64bitCR3();
2723	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2724	}
2725	else
2726	#endif
2727	{
2728	uReg = ASMGetCR3();
2729	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2730	}
2731	AssertRCReturn(rc, rc);
2732
2733	uReg = ASMGetCR4();
2734	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2735	AssertRCReturn(rc, rc);
2736	return rc;
2737	}
2738
2739
2740	#if HC_ARCH_BITS == 64
2741	/**
2742	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2743	* requirements. See hmR0VmxSaveHostSegmentRegs().
2744	*/
2745	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2746	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2747	{ \
2748	bool fValidSelector = true; \
2749	if ((selValue) & X86_SEL_LDT) \
2750	{ \
2751	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2752	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2753	} \
2754	if (fValidSelector) \
2755	{ \
2756	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2757	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2758	} \
2759	(selValue) = 0; \
2760	}
2761	#endif
2762
2763
2764	/**
2765	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2766	* the host-state area in the VMCS.
2767	*
2768	* @returns VBox status code.
2769	* @param pVM Pointer to the VM.
2770	* @param pVCpu Pointer to the VMCPU.
2771	*/
2772	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2773	{
2774	NOREF(pVM);
2775	int rc = VERR_INTERNAL_ERROR_5;
2776
2777	/*
2778	* Quick fix for regression #7240. Restore the host state if we've messed
2779	* it up already, otherwise all we'll get it all wrong below!
2780	*/
2781	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2782	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2783	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2784
2785	/*
2786	* Host DS, ES, FS and GS segment registers.
2787	*/
2788	#if HC_ARCH_BITS == 64
2789	RTSEL uSelDS = ASMGetDS();
2790	RTSEL uSelES = ASMGetES();
2791	RTSEL uSelFS = ASMGetFS();
2792	RTSEL uSelGS = ASMGetGS();
2793	#else
2794	RTSEL uSelDS = 0;
2795	RTSEL uSelES = 0;
2796	RTSEL uSelFS = 0;
2797	RTSEL uSelGS = 0;
2798	#endif
2799
2800	/* Recalculate which host-state bits need to be manually restored. */
2801	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2802
2803	/*
2804	* Host CS and SS segment registers.
2805	*/
2806	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2807	RTSEL uSelCS;
2808	RTSEL uSelSS;
2809	if (HMVMX_IS_64BIT_HOST_MODE())
2810	{
2811	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2812	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2813	}
2814	else
2815	{
2816	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2817	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2818	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2819	}
2820	#else
2821	RTSEL uSelCS = ASMGetCS();
2822	RTSEL uSelSS = ASMGetSS();
2823	#endif
2824
2825	/*
2826	* Host TR segment register.
2827	*/
2828	RTSEL uSelTR = ASMGetTR();
2829
2830	#if HC_ARCH_BITS == 64
2831	/*
2832	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2833	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2834	*/
2835	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2836	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2837	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2838	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2839	# undef VMXLOCAL_ADJUST_HOST_SEG
2840	#endif
2841
2842	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2843	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2844	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2845	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2846	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2847	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2848	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2849	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2850	Assert(uSelCS);
2851	Assert(uSelTR);
2852
2853	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2854	#if 0
2855	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2856	Assert(uSelSS != 0);
2857	#endif
2858
2859	/* Write these host selector fields into the host-state area in the VMCS. */
2860	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2861	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2862	#if HC_ARCH_BITS == 64
2863	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2864	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2865	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2866	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2867	#endif
2868	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2869
2870	/*
2871	* Host GDTR and IDTR.
2872	*/
2873	RTGDTR Gdtr;
2874	RT_ZERO(Gdtr);
2875	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2876	if (HMVMX_IS_64BIT_HOST_MODE())
2877	{
2878	X86XDTR64 Gdtr64;
2879	X86XDTR64 Idtr64;
2880	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2881	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2882	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2883
2884	Gdtr.cbGdt = Gdtr64.cb;
2885	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2886	}
2887	else
2888	#endif
2889	{
2890	RTIDTR Idtr;
2891	ASMGetGDTR(&Gdtr);
2892	ASMGetIDTR(&Idtr);
2893	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2894	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2895
2896	#if HC_ARCH_BITS == 64
2897	/*
2898	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2899	* maximum limit (0xffff) on every VM-exit.
2900	*/
2901	if (Gdtr.cbGdt != 0xffff)
2902	{
2903	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2904	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2905	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2906	}
2907
2908	/*
2909	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2910	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
2911	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
2912	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2913	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
2914	* hosts where we are pretty sure it won't cause trouble.
2915	*/
2916	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
2917	if (Idtr.cbIdt < 0x0fff)
2918	# else
2919	if (Idtr.cbIdt != 0xffff)
2920	# endif
2921	{
2922	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2923	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2924	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2925	}
2926	#endif
2927	}
2928
2929	/*
2930	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2931	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2932	*/
2933	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2934	{
2935	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2936	return VERR_VMX_INVALID_HOST_STATE;
2937	}
2938
2939	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2940	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2941	if (HMVMX_IS_64BIT_HOST_MODE())
2942	{
2943	/* We need the 64-bit TR base for hybrid darwin. */
2944	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2945	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2946	}
2947	else
2948	#endif
2949	{
2950	uintptr_t uTRBase;
2951	#if HC_ARCH_BITS == 64
2952	uTRBase = X86DESC64_BASE(pDesc);
2953
2954	/*
2955	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2956	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2957	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2958	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2959	*
2960	* [1] See Intel spec. 3.5 "System Descriptor Types".
2961	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2962	*/
2963	Assert(pDesc->System.u4Type == 11);
2964	if ( pDesc->System.u16LimitLow != 0x67
2965	\|\| pDesc->System.u4LimitHigh)
2966	{
2967	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2968	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2969
2970	/* Store the GDTR here as we need it while restoring TR. */
2971	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2972	}
2973	#else
2974	uTRBase = X86DESC_BASE(pDesc);
2975	#endif
2976	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2977	}
2978	AssertRCReturn(rc, rc);
2979
2980	/*
2981	* Host FS base and GS base.
2982	*/
2983	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2984	if (HMVMX_IS_64BIT_HOST_MODE())
2985	{
2986	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2987	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2988	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2989	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2990
2991	# if HC_ARCH_BITS == 64
2992	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2993	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2994	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2995	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2996	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2997	# endif
2998	}
2999	#endif
3000	return rc;
3001	}
3002
3003
3004	/**
3005	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3006	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3007	* the host after every successful VM-exit.
3008	*
3009	* @returns VBox status code.
3010	* @param pVM Pointer to the VM.
3011	* @param pVCpu Pointer to the VMCPU.
3012	*
3013	* @remarks No-long-jump zone!!!
3014	*/
3015	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3016	{
3017	NOREF(pVM);
3018
3019	AssertPtr(pVCpu);
3020	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3021
3022	int rc = VINF_SUCCESS;
3023	#if HC_ARCH_BITS == 64
3024	if (pVM->hm.s.fAllow64BitGuests)
3025	hmR0VmxLazySaveHostMsrs(pVCpu);
3026	#endif
3027
3028	if (pVCpu->hm.s.vmx.cMsrs > 0)
3029	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
3030
3031	/*
3032	* Host Sysenter MSRs.
3033	*/
3034	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3035	AssertRCReturn(rc, rc);
3036	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3037	if (HMVMX_IS_64BIT_HOST_MODE())
3038	{
3039	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3040	AssertRCReturn(rc, rc);
3041	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3042	}
3043	else
3044	{
3045	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3046	AssertRCReturn(rc, rc);
3047	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3048	}
3049	#elif HC_ARCH_BITS == 32
3050	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3051	AssertRCReturn(rc, rc);
3052	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3053	#else
3054	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3055	AssertRCReturn(rc, rc);
3056	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3057	#endif
3058	AssertRCReturn(rc, rc);
3059
3060	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3061	* hmR0VmxSetupExitCtls() !! */
3062	return rc;
3063	}
3064
3065
3066	/**
3067	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3068	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3069	* controls".
3070	*
3071	* @returns VBox status code.
3072	* @param pVCpu Pointer to the VMCPU.
3073	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3074	* out-of-sync. Make sure to update the required fields
3075	* before using them.
3076	*
3077	* @remarks No-long-jump zone!!!
3078	*/
3079	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3080	{
3081	int rc = VINF_SUCCESS;
3082	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3083	{
3084	PVM pVM = pVCpu->CTX_SUFF(pVM);
3085	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3086	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3087
3088	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3089	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3090
3091	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3092	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3093	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3094	else
3095	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3096
3097	/*
3098	* The following should -not- be set (since we're not in SMM mode):
3099	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3100	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3101	*/
3102
3103	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3104	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3105	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3106
3107	if ((val & zap) != val)
3108	{
3109	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3110	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3111	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3112	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3113	}
3114
3115	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3116	AssertRCReturn(rc, rc);
3117
3118	/* Update VCPU with the currently set VM-exit controls. */
3119	pVCpu->hm.s.vmx.u32EntryCtls = val;
3120	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3121	}
3122	return rc;
3123	}
3124
3125
3126	/**
3127	* Sets up the VM-exit controls in the VMCS.
3128	*
3129	* @returns VBox status code.
3130	* @param pVM Pointer to the VM.
3131	* @param pVCpu Pointer to the VMCPU.
3132	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3133	* out-of-sync. Make sure to update the required fields
3134	* before using them.
3135	*
3136	* @remarks requires EFER.
3137	*/
3138	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3139	{
3140	NOREF(pMixedCtx);
3141
3142	int rc = VINF_SUCCESS;
3143	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3144	{
3145	PVM pVM = pVCpu->CTX_SUFF(pVM);
3146	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3147	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3148
3149	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3150	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3151
3152	/*
3153	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3154	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3155	*/
3156	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3157	if (HMVMX_IS_64BIT_HOST_MODE())
3158	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3159	else
3160	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3161	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3162	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3163	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3164	else
3165	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3166	#endif
3167
3168	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3169	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3170
3171	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3172	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3173	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3174	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3175	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3176
3177	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3178	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3179
3180	if ((val & zap) != val)
3181	{
3182	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3183	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3184	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3185	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3186	}
3187
3188	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3189	AssertRCReturn(rc, rc);
3190
3191	/* Update VCPU with the currently set VM-exit controls. */
3192	pVCpu->hm.s.vmx.u32ExitCtls = val;
3193	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3194	}
3195	return rc;
3196	}
3197
3198
3199	/**
3200	* Loads the guest APIC and related state.
3201	*
3202	* @returns VBox status code.
3203	* @param pVM Pointer to the VM.
3204	* @param pVCpu Pointer to the VMCPU.
3205	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3206	* out-of-sync. Make sure to update the required fields
3207	* before using them.
3208	*/
3209	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3210	{
3211	NOREF(pMixedCtx);
3212
3213	int rc = VINF_SUCCESS;
3214	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3215	{
3216	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3217	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3218	{
3219	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3220
3221	bool fPendingIntr = false;
3222	uint8_t u8Tpr = 0;
3223	uint8_t u8PendingIntr = 0;
3224	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3225	AssertRCReturn(rc, rc);
3226
3227	/*
3228	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3229	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3230	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3231	* the interrupt when we VM-exit for other reasons.
3232	*/
3233	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3234	uint32_t u32TprThreshold = 0;
3235	if (fPendingIntr)
3236	{
3237	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3238	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3239	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3240	if (u8PendingPriority <= u8TprPriority)
3241	u32TprThreshold = u8PendingPriority;
3242	else
3243	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3244	}
3245	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3246
3247	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3248	AssertRCReturn(rc, rc);
3249	}
3250
3251	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3252	}
3253	return rc;
3254	}
3255
3256
3257	/**
3258	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3259	*
3260	* @returns Guest's interruptibility-state.
3261	* @param pVCpu Pointer to the VMCPU.
3262	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3263	* out-of-sync. Make sure to update the required fields
3264	* before using them.
3265	*
3266	* @remarks No-long-jump zone!!!
3267	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3268	*/
3269	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3270	{
3271	/*
3272	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3273	* inhibit interrupts or clear any existing interrupt-inhibition.
3274	*/
3275	uint32_t uIntrState = 0;
3276	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3277	{
3278	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3279	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3280	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3281	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3282	{
3283	/*
3284	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3285	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3286	*/
3287	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3288	}
3289	else if (pMixedCtx->eflags.Bits.u1IF)
3290	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3291	else
3292	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3293	}
3294	return uIntrState;
3295	}
3296
3297
3298	/**
3299	* Loads the guest's interruptibility-state into the guest-state area in the
3300	* VMCS.
3301	*
3302	* @returns VBox status code.
3303	* @param pVCpu Pointer to the VMCPU.
3304	* @param uIntrState The interruptibility-state to set.
3305	*/
3306	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3307	{
3308	NOREF(pVCpu);
3309	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3310	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3311	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3312	AssertRCReturn(rc, rc);
3313	return rc;
3314	}
3315
3316
3317	/**
3318	* Loads the guest's RIP into the guest-state area in the VMCS.
3319	*
3320	* @returns VBox status code.
3321	* @param pVCpu Pointer to the VMCPU.
3322	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3323	* out-of-sync. Make sure to update the required fields
3324	* before using them.
3325	*
3326	* @remarks No-long-jump zone!!!
3327	*/
3328	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3329	{
3330	int rc = VINF_SUCCESS;
3331	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3332	{
3333	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3334	AssertRCReturn(rc, rc);
3335
3336	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3337	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3338	}
3339	return rc;
3340	}
3341
3342
3343	/**
3344	* Loads the guest's RSP into the guest-state area in the VMCS.
3345	*
3346	* @returns VBox status code.
3347	* @param pVCpu Pointer to the VMCPU.
3348	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3349	* out-of-sync. Make sure to update the required fields
3350	* before using them.
3351	*
3352	* @remarks No-long-jump zone!!!
3353	*/
3354	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3355	{
3356	int rc = VINF_SUCCESS;
3357	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3358	{
3359	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3360	AssertRCReturn(rc, rc);
3361
3362	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3363	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3364	}
3365	return rc;
3366	}
3367
3368
3369	/**
3370	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3371	*
3372	* @returns VBox status code.
3373	* @param pVCpu Pointer to the VMCPU.
3374	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3375	* out-of-sync. Make sure to update the required fields
3376	* before using them.
3377	*
3378	* @remarks No-long-jump zone!!!
3379	*/
3380	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3381	{
3382	int rc = VINF_SUCCESS;
3383	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3384	{
3385	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3386	Let us assert it as such and use 32-bit VMWRITE. */
3387	Assert(!(pMixedCtx->rflags.u64 >> 32));
3388	X86EFLAGS Eflags = pMixedCtx->eflags;
3389	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3390	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3391
3392	/*
3393	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3394	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3395	*/
3396	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3397	{
3398	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3399	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3400	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3401	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3402	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3403	}
3404
3405	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3406	AssertRCReturn(rc, rc);
3407
3408	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3409	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3410	}
3411	return rc;
3412	}
3413
3414
3415	/**
3416	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3417	*
3418	* @returns VBox status code.
3419	* @param pVCpu Pointer to the VMCPU.
3420	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3421	* out-of-sync. Make sure to update the required fields
3422	* before using them.
3423	*
3424	* @remarks No-long-jump zone!!!
3425	*/
3426	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3427	{
3428	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3429	AssertRCReturn(rc, rc);
3430	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3431	AssertRCReturn(rc, rc);
3432	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3433	AssertRCReturn(rc, rc);
3434	return rc;
3435	}
3436
3437
3438	/**
3439	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3440	* CR0 is partially shared with the host and we have to consider the FPU bits.
3441	*
3442	* @returns VBox status code.
3443	* @param pVM Pointer to the VM.
3444	* @param pVCpu Pointer to the VMCPU.
3445	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3446	* out-of-sync. Make sure to update the required fields
3447	* before using them.
3448	*
3449	* @remarks No-long-jump zone!!!
3450	*/
3451	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3452	{
3453	/*
3454	* Guest CR0.
3455	* Guest FPU.
3456	*/
3457	int rc = VINF_SUCCESS;
3458	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3459	{
3460	Assert(!(pMixedCtx->cr0 >> 32));
3461	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3462	PVM pVM = pVCpu->CTX_SUFF(pVM);
3463
3464	/* The guest's view (read access) of its CR0 is unblemished. */
3465	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3466	AssertRCReturn(rc, rc);
3467	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3468
3469	/* Setup VT-x's view of the guest CR0. */
3470	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3471	if (pVM->hm.s.fNestedPaging)
3472	{
3473	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3474	{
3475	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3476	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3477	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3478	}
3479	else
3480	{
3481	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3482	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3483	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3484	}
3485
3486	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3487	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3488	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3489
3490	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3491	AssertRCReturn(rc, rc);
3492	}
3493	else
3494	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3495
3496	/*
3497	* Guest FPU bits.
3498	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3499	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3500	*/
3501	u32GuestCR0 \|= X86_CR0_NE;
3502	bool fInterceptNM = false;
3503	if (CPUMIsGuestFPUStateActive(pVCpu))
3504	{
3505	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3506	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3507	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3508	}
3509	else
3510	{
3511	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3512	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3513	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3514	}
3515
3516	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3517	bool fInterceptMF = false;
3518	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3519	fInterceptMF = true;
3520
3521	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3522	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3523	{
3524	Assert(PDMVmmDevHeapIsEnabled(pVM));
3525	Assert(pVM->hm.s.vmx.pRealModeTSS);
3526	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3527	fInterceptNM = true;
3528	fInterceptMF = true;
3529	}
3530	else
3531	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3532
3533	if (fInterceptNM)
3534	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3535	else
3536	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3537
3538	if (fInterceptMF)
3539	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3540	else
3541	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3542
3543	/* Additional intercepts for debugging, define these yourself explicitly. */
3544	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3545	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3546	\| RT_BIT(X86_XCPT_BP)
3547	\| RT_BIT(X86_XCPT_DB)
3548	\| RT_BIT(X86_XCPT_DE)
3549	\| RT_BIT(X86_XCPT_NM)
3550	\| RT_BIT(X86_XCPT_UD)
3551	\| RT_BIT(X86_XCPT_NP)
3552	\| RT_BIT(X86_XCPT_SS)
3553	\| RT_BIT(X86_XCPT_GP)
3554	\| RT_BIT(X86_XCPT_PF)
3555	\| RT_BIT(X86_XCPT_MF)
3556	;
3557	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3558	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3559	#endif
3560
3561	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3562
3563	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3564	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3565	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3566	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3567	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3568	else
3569	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3570
3571	u32GuestCR0 \|= uSetCR0;
3572	u32GuestCR0 &= uZapCR0;
3573	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3574
3575	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3576	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3577	AssertRCReturn(rc, rc);
3578	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3579	AssertRCReturn(rc, rc);
3580	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3581
3582	/*
3583	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3584	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3585	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3586	*/
3587	uint32_t u32CR0Mask = 0;
3588	u32CR0Mask = X86_CR0_PE
3589	\| X86_CR0_NE
3590	\| X86_CR0_WP
3591	\| X86_CR0_PG
3592	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3593	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3594	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3595
3596	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3597	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3598	* and @bugref{6944}. */
3599	#if 0
3600	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3601	u32CR0Mask &= ~X86_CR0_PE;
3602	#endif
3603	if (pVM->hm.s.fNestedPaging)
3604	u32CR0Mask &= ~X86_CR0_WP;
3605
3606	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3607	if (fInterceptNM)
3608	{
3609	u32CR0Mask \|= X86_CR0_TS
3610	\| X86_CR0_MP;
3611	}
3612
3613	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3614	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3615	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3616	AssertRCReturn(rc, rc);
3617	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3618
3619	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3620	}
3621	return rc;
3622	}
3623
3624
3625	/**
3626	* Loads the guest control registers (CR3, CR4) into the guest-state area
3627	* in the VMCS.
3628	*
3629	* @returns VBox status code.
3630	* @param pVM Pointer to the VM.
3631	* @param pVCpu Pointer to the VMCPU.
3632	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3633	* out-of-sync. Make sure to update the required fields
3634	* before using them.
3635	*
3636	* @remarks No-long-jump zone!!!
3637	*/
3638	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3639	{
3640	int rc = VINF_SUCCESS;
3641	PVM pVM = pVCpu->CTX_SUFF(pVM);
3642
3643	/*
3644	* Guest CR2.
3645	* It's always loaded in the assembler code. Nothing to do here.
3646	*/
3647
3648	/*
3649	* Guest CR3.
3650	*/
3651	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3652	{
3653	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3654	if (pVM->hm.s.fNestedPaging)
3655	{
3656	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3657
3658	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3659	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3660	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3661	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3662
3663	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3664	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3665	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3666
3667	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3668	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3669	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3670	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3671
3672	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3673	AssertRCReturn(rc, rc);
3674	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3675
3676	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3677	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3678	{
3679	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3680	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3681	{
3682	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3683	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3684	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3685	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3686	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3687	}
3688
3689	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3690	have Unrestricted Execution to handle the guest when it's not using paging. */
3691	GCPhysGuestCR3 = pMixedCtx->cr3;
3692	}
3693	else
3694	{
3695	/*
3696	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3697	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3698	* EPT takes care of translating it to host-physical addresses.
3699	*/
3700	RTGCPHYS GCPhys;
3701	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3702	Assert(PDMVmmDevHeapIsEnabled(pVM));
3703
3704	/* We obtain it here every time as the guest could have relocated this PCI region. */
3705	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3706	AssertRCReturn(rc, rc);
3707
3708	GCPhysGuestCR3 = GCPhys;
3709	}
3710
3711	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3712	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3713	}
3714	else
3715	{
3716	/* Non-nested paging case, just use the hypervisor's CR3. */
3717	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3718
3719	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3720	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3721	}
3722	AssertRCReturn(rc, rc);
3723
3724	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3725	}
3726
3727	/*
3728	* Guest CR4.
3729	*/
3730	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3731	{
3732	Assert(!(pMixedCtx->cr4 >> 32));
3733	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3734
3735	/* The guest's view of its CR4 is unblemished. */
3736	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3737	AssertRCReturn(rc, rc);
3738	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3739
3740	/* Setup VT-x's view of the guest CR4. */
3741	/*
3742	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3743	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3744	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3745	*/
3746	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3747	{
3748	Assert(pVM->hm.s.vmx.pRealModeTSS);
3749	Assert(PDMVmmDevHeapIsEnabled(pVM));
3750	u32GuestCR4 &= ~X86_CR4_VME;
3751	}
3752
3753	if (pVM->hm.s.fNestedPaging)
3754	{
3755	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3756	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3757	{
3758	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3759	u32GuestCR4 \|= X86_CR4_PSE;
3760	/* Our identity mapping is a 32-bit page directory. */
3761	u32GuestCR4 &= ~X86_CR4_PAE;
3762	}
3763	/* else use guest CR4.*/
3764	}
3765	else
3766	{
3767	/*
3768	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3769	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3770	*/
3771	switch (pVCpu->hm.s.enmShadowMode)
3772	{
3773	case PGMMODE_REAL: /* Real-mode. */
3774	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3775	case PGMMODE_32_BIT: /* 32-bit paging. */
3776	{
3777	u32GuestCR4 &= ~X86_CR4_PAE;
3778	break;
3779	}
3780
3781	case PGMMODE_PAE: /* PAE paging. */
3782	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3783	{
3784	u32GuestCR4 \|= X86_CR4_PAE;
3785	break;
3786	}
3787
3788	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3789	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3790	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3791	break;
3792	#endif
3793	default:
3794	AssertFailed();
3795	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3796	}
3797	}
3798
3799	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3800	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3801	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3802	u32GuestCR4 \|= uSetCR4;
3803	u32GuestCR4 &= uZapCR4;
3804
3805	/* Write VT-x's view of the guest CR4 into the VMCS. */
3806	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3807	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3808	AssertRCReturn(rc, rc);
3809
3810	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3811	uint32_t u32CR4Mask = 0;
3812	u32CR4Mask = X86_CR4_VME
3813	\| X86_CR4_PAE
3814	\| X86_CR4_PGE
3815	\| X86_CR4_PSE
3816	\| X86_CR4_VMXE;
3817	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3818	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3819	AssertRCReturn(rc, rc);
3820
3821	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3822	}
3823	return rc;
3824	}
3825
3826
3827	/**
3828	* Loads the guest debug registers into the guest-state area in the VMCS.
3829	* This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3830	*
3831	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3832	*
3833	* @returns VBox status code.
3834	* @param pVCpu Pointer to the VMCPU.
3835	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3836	* out-of-sync. Make sure to update the required fields
3837	* before using them.
3838	*
3839	* @remarks No-long-jump zone!!!
3840	*/
3841	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3842	{
3843	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3844	return VINF_SUCCESS;
3845
3846	#ifdef VBOX_STRICT
3847	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3848	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3849	{
3850	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3851	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3852	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3853	}
3854	#endif
3855
3856	int rc;
3857	PVM pVM = pVCpu->CTX_SUFF(pVM);
3858	bool fInterceptDB = false;
3859	bool fInterceptMovDRx = false;
3860	if ( pVCpu->hm.s.fSingleInstruction
3861	\|\| DBGFIsStepping(pVCpu))
3862	{
3863	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3864	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3865	{
3866	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3867	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3868	AssertRCReturn(rc, rc);
3869	Assert(fInterceptDB == false);
3870	}
3871	else
3872	{
3873	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3874	pVCpu->hm.s.fClearTrapFlag = true;
3875	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3876	fInterceptDB = true;
3877	}
3878	}
3879
3880	if ( fInterceptDB
3881	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3882	{
3883	/*
3884	* Use the combined guest and host DRx values found in the hypervisor
3885	* register set because the debugger has breakpoints active or someone
3886	* is single stepping on the host side without a monitor trap flag.
3887	*
3888	* Note! DBGF expects a clean DR6 state before executing guest code.
3889	*/
3890	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3891	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3892	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3893	{
3894	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3895	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3896	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3897	}
3898	else
3899	#endif
3900	if (!CPUMIsHyperDebugStateActive(pVCpu))
3901	{
3902	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3903	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3904	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3905	}
3906
3907	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3908	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3909	AssertRCReturn(rc, rc);
3910
3911	pVCpu->hm.s.fUsingHyperDR7 = true;
3912	fInterceptDB = true;
3913	fInterceptMovDRx = true;
3914	}
3915	else
3916	{
3917	/*
3918	* If the guest has enabled debug registers, we need to load them prior to
3919	* executing guest code so they'll trigger at the right time.
3920	*/
3921	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3922	{
3923	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3924	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3925	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3926	{
3927	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3928	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3929	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3930	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3931	}
3932	else
3933	#endif
3934	if (!CPUMIsGuestDebugStateActive(pVCpu))
3935	{
3936	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3937	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3938	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3939	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3940	}
3941	Assert(!fInterceptDB);
3942	Assert(!fInterceptMovDRx);
3943	}
3944	/*
3945	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3946	* must intercept #DB in order to maintain a correct DR6 guest value.
3947	*/
3948	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3949	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3950	&& !CPUMIsGuestDebugStateActive(pVCpu))
3951	#else
3952	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3953	#endif
3954	{
3955	fInterceptMovDRx = true;
3956	fInterceptDB = true;
3957	}
3958
3959	/* Update guest DR7. */
3960	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3961	AssertRCReturn(rc, rc);
3962
3963	pVCpu->hm.s.fUsingHyperDR7 = false;
3964	}
3965
3966	/*
3967	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3968	*/
3969	if (fInterceptDB)
3970	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3971	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3972	{
3973	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3974	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3975	#endif
3976	}
3977	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3978	AssertRCReturn(rc, rc);
3979
3980	/*
3981	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3982	*/
3983	if (fInterceptMovDRx)
3984	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3985	else
3986	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3987	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3988	AssertRCReturn(rc, rc);
3989
3990	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3991	return VINF_SUCCESS;
3992	}
3993
3994
3995	#ifdef VBOX_STRICT
3996	/**
3997	* Strict function to validate segment registers.
3998	*
3999	* @remarks ASSUMES CR0 is up to date.
4000	*/
4001	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4002	{
4003	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4004	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4005	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4006	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4007	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4008	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4009	{
4010	/* Protected mode checks */
4011	/* CS */
4012	Assert(pCtx->cs.Attr.n.u1Present);
4013	Assert(!(pCtx->cs.Attr.u & 0xf00));
4014	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4015	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4016	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4017	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4018	\|\| (pCtx->cs.Attr.n.u1Granularity));
4019	/* CS cannot be loaded with NULL in protected mode. */
4020	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4021	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4022	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4023	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4024	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4025	else
4026	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4027	/* SS */
4028	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4029	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4030	if ( !(pCtx->cr0 & X86_CR0_PE)
4031	\|\| pCtx->cs.Attr.n.u4Type == 3)
4032	{
4033	Assert(!pCtx->ss.Attr.n.u2Dpl);
4034	}
4035	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4036	{
4037	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4038	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4039	Assert(pCtx->ss.Attr.n.u1Present);
4040	Assert(!(pCtx->ss.Attr.u & 0xf00));
4041	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4042	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4043	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4044	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4045	\|\| (pCtx->ss.Attr.n.u1Granularity));
4046	}
4047	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4048	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4049	{
4050	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4051	Assert(pCtx->ds.Attr.n.u1Present);
4052	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4053	Assert(!(pCtx->ds.Attr.u & 0xf00));
4054	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4055	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4056	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4057	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4058	\|\| (pCtx->ds.Attr.n.u1Granularity));
4059	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4060	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4061	}
4062	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4063	{
4064	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4065	Assert(pCtx->es.Attr.n.u1Present);
4066	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4067	Assert(!(pCtx->es.Attr.u & 0xf00));
4068	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4069	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4070	\|\| !(pCtx->es.Attr.n.u1Granularity));
4071	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4072	\|\| (pCtx->es.Attr.n.u1Granularity));
4073	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4074	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4075	}
4076	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4077	{
4078	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4079	Assert(pCtx->fs.Attr.n.u1Present);
4080	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4081	Assert(!(pCtx->fs.Attr.u & 0xf00));
4082	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4083	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4084	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4085	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4086	\|\| (pCtx->fs.Attr.n.u1Granularity));
4087	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4088	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4089	}
4090	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4091	{
4092	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4093	Assert(pCtx->gs.Attr.n.u1Present);
4094	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4095	Assert(!(pCtx->gs.Attr.u & 0xf00));
4096	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4097	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4098	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4099	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4100	\|\| (pCtx->gs.Attr.n.u1Granularity));
4101	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4102	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4103	}
4104	/* 64-bit capable CPUs. */
4105	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4106	if (HMVMX_IS_64BIT_HOST_MODE())
4107	{
4108	Assert(!(pCtx->cs.u64Base >> 32));
4109	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4110	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4111	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4112	}
4113	# endif
4114	}
4115	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4116	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4117	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4118	{
4119	/* Real and v86 mode checks. */
4120	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4121	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4122	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4123	{
4124	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4125	}
4126	else
4127	{
4128	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4129	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4130	}
4131
4132	/* CS */
4133	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4134	Assert(pCtx->cs.u32Limit == 0xffff);
4135	Assert(u32CSAttr == 0xf3);
4136	/* SS */
4137	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4138	Assert(pCtx->ss.u32Limit == 0xffff);
4139	Assert(u32SSAttr == 0xf3);
4140	/* DS */
4141	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4142	Assert(pCtx->ds.u32Limit == 0xffff);
4143	Assert(u32DSAttr == 0xf3);
4144	/* ES */
4145	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4146	Assert(pCtx->es.u32Limit == 0xffff);
4147	Assert(u32ESAttr == 0xf3);
4148	/* FS */
4149	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4150	Assert(pCtx->fs.u32Limit == 0xffff);
4151	Assert(u32FSAttr == 0xf3);
4152	/* GS */
4153	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4154	Assert(pCtx->gs.u32Limit == 0xffff);
4155	Assert(u32GSAttr == 0xf3);
4156	/* 64-bit capable CPUs. */
4157	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4158	if (HMVMX_IS_64BIT_HOST_MODE())
4159	{
4160	Assert(!(pCtx->cs.u64Base >> 32));
4161	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4162	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4163	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4164	}
4165	# endif
4166	}
4167	}
4168	#endif /* VBOX_STRICT */
4169
4170
4171	/**
4172	* Writes a guest segment register into the guest-state area in the VMCS.
4173	*
4174	* @returns VBox status code.
4175	* @param pVCpu Pointer to the VMCPU.
4176	* @param idxSel Index of the selector in the VMCS.
4177	* @param idxLimit Index of the segment limit in the VMCS.
4178	* @param idxBase Index of the segment base in the VMCS.
4179	* @param idxAccess Index of the access rights of the segment in the VMCS.
4180	* @param pSelReg Pointer to the segment selector.
4181	*
4182	* @remarks No-long-jump zone!!!
4183	*/
4184	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4185	uint32_t idxAccess, PCPUMSELREG pSelReg)
4186	{
4187	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4188	AssertRCReturn(rc, rc);
4189	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4190	AssertRCReturn(rc, rc);
4191	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4192	AssertRCReturn(rc, rc);
4193
4194	uint32_t u32Access = pSelReg->Attr.u;
4195	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4196	{
4197	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4198	u32Access = 0xf3;
4199	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4200	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4201	}
4202	else
4203	{
4204	/*
4205	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4206	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4207	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4208	* loaded in protected-mode have their attribute as 0.
4209	*/
4210	if (!u32Access)
4211	u32Access = X86DESCATTR_UNUSABLE;
4212	}
4213
4214	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4215	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4216	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4217
4218	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4219	AssertRCReturn(rc, rc);
4220	return rc;
4221	}
4222
4223
4224	/**
4225	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4226	* into the guest-state area in the VMCS.
4227	*
4228	* @returns VBox status code.
4229	* @param pVM Pointer to the VM.
4230	* @param pVCPU Pointer to the VMCPU.
4231	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4232	* out-of-sync. Make sure to update the required fields
4233	* before using them.
4234	*
4235	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4236	* @remarks No-long-jump zone!!!
4237	*/
4238	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4239	{
4240	int rc = VERR_INTERNAL_ERROR_5;
4241	PVM pVM = pVCpu->CTX_SUFF(pVM);
4242
4243	/*
4244	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4245	*/
4246	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4247	{
4248	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4249	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4250	{
4251	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4252	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4253	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4254	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4255	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4256	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4257	}
4258
4259	#ifdef VBOX_WITH_REM
4260	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4261	{
4262	Assert(pVM->hm.s.vmx.pRealModeTSS);
4263	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4264	if ( pVCpu->hm.s.vmx.fWasInRealMode
4265	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4266	{
4267	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4268	in real-mode (e.g. OpenBSD 4.0) */
4269	REMFlushTBs(pVM);
4270	Log4(("Load: Switch to protected mode detected!\n"));
4271	pVCpu->hm.s.vmx.fWasInRealMode = false;
4272	}
4273	}
4274	#endif
4275	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4276	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4277	AssertRCReturn(rc, rc);
4278	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4279	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4280	AssertRCReturn(rc, rc);
4281	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4282	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4283	AssertRCReturn(rc, rc);
4284	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4285	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4286	AssertRCReturn(rc, rc);
4287	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4288	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4289	AssertRCReturn(rc, rc);
4290	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4291	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4292	AssertRCReturn(rc, rc);
4293
4294	#ifdef VBOX_STRICT
4295	/* Validate. */
4296	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4297	#endif
4298
4299	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4300	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4301	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4302	}
4303
4304	/*
4305	* Guest TR.
4306	*/
4307	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4308	{
4309	/*
4310	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4311	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4312	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4313	*/
4314	uint16_t u16Sel = 0;
4315	uint32_t u32Limit = 0;
4316	uint64_t u64Base = 0;
4317	uint32_t u32AccessRights = 0;
4318
4319	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4320	{
4321	u16Sel = pMixedCtx->tr.Sel;
4322	u32Limit = pMixedCtx->tr.u32Limit;
4323	u64Base = pMixedCtx->tr.u64Base;
4324	u32AccessRights = pMixedCtx->tr.Attr.u;
4325	}
4326	else
4327	{
4328	Assert(pVM->hm.s.vmx.pRealModeTSS);
4329	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4330
4331	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4332	RTGCPHYS GCPhys;
4333	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4334	AssertRCReturn(rc, rc);
4335
4336	X86DESCATTR DescAttr;
4337	DescAttr.u = 0;
4338	DescAttr.n.u1Present = 1;
4339	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4340
4341	u16Sel = 0;
4342	u32Limit = HM_VTX_TSS_SIZE;
4343	u64Base = GCPhys; /* in real-mode phys = virt. */
4344	u32AccessRights = DescAttr.u;
4345	}
4346
4347	/* Validate. */
4348	Assert(!(u16Sel & RT_BIT(2)));
4349	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4350	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4351	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4352	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4353	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4354	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4355	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4356	Assert( (u32Limit & 0xfff) == 0xfff
4357	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4358	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4359	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4360
4361	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4362	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4363	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4364	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4365
4366	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4367	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4368	}
4369
4370	/*
4371	* Guest GDTR.
4372	*/
4373	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4374	{
4375	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4376	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4377
4378	/* Validate. */
4379	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4380
4381	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4382	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4383	}
4384
4385	/*
4386	* Guest LDTR.
4387	*/
4388	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4389	{
4390	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4391	uint32_t u32Access = 0;
4392	if (!pMixedCtx->ldtr.Attr.u)
4393	u32Access = X86DESCATTR_UNUSABLE;
4394	else
4395	u32Access = pMixedCtx->ldtr.Attr.u;
4396
4397	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4398	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4399	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4400	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4401
4402	/* Validate. */
4403	if (!(u32Access & X86DESCATTR_UNUSABLE))
4404	{
4405	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4406	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4407	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4408	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4409	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4410	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4411	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4412	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4413	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4414	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4415	}
4416
4417	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4418	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4419	}
4420
4421	/*
4422	* Guest IDTR.
4423	*/
4424	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4425	{
4426	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4427	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4428
4429	/* Validate. */
4430	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4431
4432	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4433	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4434	}
4435
4436	return VINF_SUCCESS;
4437	}
4438
4439
4440	/**
4441	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4442	* areas. These MSRs will automatically be loaded to the host CPU on every
4443	* successful VM entry and stored from the host CPU on every successful VM-exit.
4444	*
4445	* This also creates/updates MSR slots for the host MSRs. The actual host
4446	* MSR values are -not- updated here for performance reasons. See
4447	* hmR0VmxSaveHostMsrs().
4448	*
4449	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4450	*
4451	* @returns VBox status code.
4452	* @param pVCpu Pointer to the VMCPU.
4453	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4454	* out-of-sync. Make sure to update the required fields
4455	* before using them.
4456	*
4457	* @remarks No-long-jump zone!!!
4458	*/
4459	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4460	{
4461	AssertPtr(pVCpu);
4462	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4463
4464	/*
4465	* MSRs that we use the auto-load/store MSR area in the VMCS.
4466	*/
4467	PVM pVM = pVCpu->CTX_SUFF(pVM);
4468	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4469	{
4470	#if HC_ARCH_BITS == 32 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4471	if (pVM->hm.s.fAllow64BitGuests)
4472	{
4473	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4474	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4475	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4476	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4477	# ifdef DEBUG
4478	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4479	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4480	Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4481	# endif
4482	}
4483	#endif
4484	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4485	}
4486
4487	/*
4488	* Guest Sysenter MSRs.
4489	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4490	* VM-exits on WRMSRs for these MSRs.
4491	*/
4492	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4493	{
4494	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4495	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4496	}
4497
4498	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4499	{
4500	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4501	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4502	}
4503
4504	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4505	{
4506	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4507	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4508	}
4509
4510	return VINF_SUCCESS;
4511	}
4512
4513
4514	/**
4515	* Loads the guest activity state into the guest-state area in the VMCS.
4516	*
4517	* @returns VBox status code.
4518	* @param pVCpu Pointer to the VMCPU.
4519	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4520	* out-of-sync. Make sure to update the required fields
4521	* before using them.
4522	*
4523	* @remarks No-long-jump zone!!!
4524	*/
4525	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4526	{
4527	NOREF(pCtx);
4528	/** @todo See if we can make use of other states, e.g.
4529	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4530	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4531	{
4532	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4533	AssertRCReturn(rc, rc);
4534
4535	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4536	}
4537	return VINF_SUCCESS;
4538	}
4539
4540
4541	/**
4542	* Sets up the appropriate function to run guest code.
4543	*
4544	* @returns VBox status code.
4545	* @param pVCpu Pointer to the VMCPU.
4546	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4547	* out-of-sync. Make sure to update the required fields
4548	* before using them.
4549	*
4550	* @remarks No-long-jump zone!!!
4551	*/
4552	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4553	{
4554	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4555	{
4556	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4557	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4558	#endif
4559	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4560	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4561	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4562	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4563	{
4564	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4565	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4566	}
4567	#else
4568	/* 64-bit host or hybrid host. */
4569	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4570	#endif
4571	}
4572	else
4573	{
4574	/* Guest is not in long mode, use the 32-bit handler. */
4575	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4576	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4577	{
4578	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4579	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4580	}
4581	#else
4582	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4583	#endif
4584	}
4585	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4586	return VINF_SUCCESS;
4587	}
4588
4589
4590	/**
4591	* Wrapper for running the guest code in VT-x.
4592	*
4593	* @returns VBox strict status code.
4594	* @param pVM Pointer to the VM.
4595	* @param pVCpu Pointer to the VMCPU.
4596	* @param pCtx Pointer to the guest-CPU context.
4597	*
4598	* @remarks No-long-jump zone!!!
4599	*/
4600	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4601	{
4602	/*
4603	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4604	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4605	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4606	*/
4607	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4608	/** @todo Add stats for resume vs launch. */
4609	#ifdef VBOX_WITH_KERNEL_USING_XMM
4610	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4611	#else
4612	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4613	#endif
4614	}
4615
4616
4617	/**
4618	* Reports world-switch error and dumps some useful debug info.
4619	*
4620	* @param pVM Pointer to the VM.
4621	* @param pVCpu Pointer to the VMCPU.
4622	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4623	* @param pCtx Pointer to the guest-CPU context.
4624	* @param pVmxTransient Pointer to the VMX transient structure (only
4625	* exitReason updated).
4626	*/
4627	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4628	{
4629	Assert(pVM);
4630	Assert(pVCpu);
4631	Assert(pCtx);
4632	Assert(pVmxTransient);
4633	HMVMX_ASSERT_PREEMPT_SAFE();
4634
4635	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4636	switch (rcVMRun)
4637	{
4638	case VERR_VMX_INVALID_VMXON_PTR:
4639	AssertFailed();
4640	break;
4641	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4642	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4643	{
4644	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4645	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4646	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4647	AssertRC(rc);
4648
4649	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4650	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4651	Cannot do it here as we may have been long preempted. */
4652
4653	#ifdef VBOX_STRICT
4654	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4655	pVmxTransient->uExitReason));
4656	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4657	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4658	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4659	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4660	else
4661	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4662	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4663	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4664
4665	/* VMX control bits. */
4666	uint32_t u32Val;
4667	uint64_t u64Val;
4668	HMVMXHCUINTREG uHCReg;
4669	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4670	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4671	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4672	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4673	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4674	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4675	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4676	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4677	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4678	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4679	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4680	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4681	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4682	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4683	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4684	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4685	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4686	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4687	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4688	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4689	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4690	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4691	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4692	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4693	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4694	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4695	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4696	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4697	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4698	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4699	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4700	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4701	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4702	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4703	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4704	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4705	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4706	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4707	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4708	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4709	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4710	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4711
4712	/* Guest bits. */
4713	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4714	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4715	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4716	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4717	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4718	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4719	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4720	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4721
4722	/* Host bits. */
4723	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4724	Log4(("Host CR0 %#RHr\n", uHCReg));
4725	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4726	Log4(("Host CR3 %#RHr\n", uHCReg));
4727	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4728	Log4(("Host CR4 %#RHr\n", uHCReg));
4729
4730	RTGDTR HostGdtr;
4731	PCX86DESCHC pDesc;
4732	ASMGetGDTR(&HostGdtr);
4733	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4734	Log4(("Host CS %#08x\n", u32Val));
4735	if (u32Val < HostGdtr.cbGdt)
4736	{
4737	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4738	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4739	}
4740
4741	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4742	Log4(("Host DS %#08x\n", u32Val));
4743	if (u32Val < HostGdtr.cbGdt)
4744	{
4745	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4746	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4747	}
4748
4749	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4750	Log4(("Host ES %#08x\n", u32Val));
4751	if (u32Val < HostGdtr.cbGdt)
4752	{
4753	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4754	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4755	}
4756
4757	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4758	Log4(("Host FS %#08x\n", u32Val));
4759	if (u32Val < HostGdtr.cbGdt)
4760	{
4761	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4762	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4763	}
4764
4765	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4766	Log4(("Host GS %#08x\n", u32Val));
4767	if (u32Val < HostGdtr.cbGdt)
4768	{
4769	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4770	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4771	}
4772
4773	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4774	Log4(("Host SS %#08x\n", u32Val));
4775	if (u32Val < HostGdtr.cbGdt)
4776	{
4777	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4778	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4779	}
4780
4781	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4782	Log4(("Host TR %#08x\n", u32Val));
4783	if (u32Val < HostGdtr.cbGdt)
4784	{
4785	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4786	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4787	}
4788
4789	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4790	Log4(("Host TR Base %#RHv\n", uHCReg));
4791	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4792	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4793	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4794	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4795	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4796	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4797	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4798	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4799	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4800	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4801	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4802	Log4(("Host RSP %#RHv\n", uHCReg));
4803	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4804	Log4(("Host RIP %#RHv\n", uHCReg));
4805	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4806	if (HMVMX_IS_64BIT_HOST_MODE())
4807	{
4808	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4809	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4810	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4811	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4812	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4813	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4814	}
4815	# endif
4816	#endif /* VBOX_STRICT */
4817	break;
4818	}
4819
4820	default:
4821	/* Impossible */
4822	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4823	break;
4824	}
4825	NOREF(pVM); NOREF(pCtx);
4826	}
4827
4828
4829	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4830	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4831	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4832	#endif
4833	#ifdef VBOX_STRICT
4834	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4835	{
4836	switch (idxField)
4837	{
4838	case VMX_VMCS_GUEST_RIP:
4839	case VMX_VMCS_GUEST_RSP:
4840	case VMX_VMCS_GUEST_SYSENTER_EIP:
4841	case VMX_VMCS_GUEST_SYSENTER_ESP:
4842	case VMX_VMCS_GUEST_GDTR_BASE:
4843	case VMX_VMCS_GUEST_IDTR_BASE:
4844	case VMX_VMCS_GUEST_CS_BASE:
4845	case VMX_VMCS_GUEST_DS_BASE:
4846	case VMX_VMCS_GUEST_ES_BASE:
4847	case VMX_VMCS_GUEST_FS_BASE:
4848	case VMX_VMCS_GUEST_GS_BASE:
4849	case VMX_VMCS_GUEST_SS_BASE:
4850	case VMX_VMCS_GUEST_LDTR_BASE:
4851	case VMX_VMCS_GUEST_TR_BASE:
4852	case VMX_VMCS_GUEST_CR3:
4853	return true;
4854	}
4855	return false;
4856	}
4857
4858	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4859	{
4860	switch (idxField)
4861	{
4862	/* Read-only fields. */
4863	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4864	return true;
4865	}
4866	/* Remaining readable fields should also be writable. */
4867	return hmR0VmxIsValidWriteField(idxField);
4868	}
4869	#endif /* VBOX_STRICT */
4870
4871
4872	/**
4873	* Executes the specified handler in 64-bit mode.
4874	*
4875	* @returns VBox status code.
4876	* @param pVM Pointer to the VM.
4877	* @param pVCpu Pointer to the VMCPU.
4878	* @param pCtx Pointer to the guest CPU context.
4879	* @param enmOp The operation to perform.
4880	* @param cbParam Number of parameters.
4881	* @param paParam Array of 32-bit parameters.
4882	*/
4883	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4884	uint32_t *paParam)
4885	{
4886	int rc, rc2;
4887	PHMGLOBALCPUINFO pCpu;
4888	RTHCPHYS HCPhysCpuPage;
4889	RTCCUINTREG uOldEflags;
4890
4891	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4892	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4893	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4894	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4895
4896	#ifdef VBOX_STRICT
4897	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4898	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4899
4900	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4901	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4902	#endif
4903
4904	/* Disable interrupts. */
4905	uOldEflags = ASMIntDisableFlags();
4906
4907	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4908	RTCPUID idHostCpu = RTMpCpuId();
4909	CPUMR0SetLApic(pVCpu, idHostCpu);
4910	#endif
4911
4912	pCpu = HMR0GetCurrentCpu();
4913	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4914
4915	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4916	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4917
4918	/* Leave VMX Root Mode. */
4919	VMXDisable();
4920
4921	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4922
4923	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4924	CPUMSetHyperEIP(pVCpu, enmOp);
4925	for (int i = (int)cbParam - 1; i >= 0; i--)
4926	CPUMPushHyper(pVCpu, paParam[i]);
4927
4928	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4929
4930	/* Call the switcher. */
4931	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4932	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4933
4934	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4935	/* Make sure the VMX instructions don't cause #UD faults. */
4936	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4937
4938	/* Re-enter VMX Root Mode */
4939	rc2 = VMXEnable(HCPhysCpuPage);
4940	if (RT_FAILURE(rc2))
4941	{
4942	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4943	ASMSetFlags(uOldEflags);
4944	return rc2;
4945	}
4946
4947	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4948	AssertRC(rc2);
4949	Assert(!(ASMGetFlags() & X86_EFL_IF));
4950	ASMSetFlags(uOldEflags);
4951	return rc;
4952	}
4953
4954
4955	/**
4956	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4957	* supporting 64-bit guests.
4958	*
4959	* @returns VBox status code.
4960	* @param fResume Whether to VMLAUNCH or VMRESUME.
4961	* @param pCtx Pointer to the guest-CPU context.
4962	* @param pCache Pointer to the VMCS cache.
4963	* @param pVM Pointer to the VM.
4964	* @param pVCpu Pointer to the VMCPU.
4965	*/
4966	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4967	{
4968	uint32_t aParam[6];
4969	PHMGLOBALCPUINFO pCpu = NULL;
4970	RTHCPHYS HCPhysCpuPage = 0;
4971	int rc = VERR_INTERNAL_ERROR_5;
4972
4973	pCpu = HMR0GetCurrentCpu();
4974	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4975
4976	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4977	pCache->uPos = 1;
4978	pCache->interPD = PGMGetInterPaeCR3(pVM);
4979	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4980	#endif
4981
4982	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4983	pCache->TestIn.HCPhysCpuPage = 0;
4984	pCache->TestIn.HCPhysVmcs = 0;
4985	pCache->TestIn.pCache = 0;
4986	pCache->TestOut.HCPhysVmcs = 0;
4987	pCache->TestOut.pCache = 0;
4988	pCache->TestOut.pCtx = 0;
4989	pCache->TestOut.eflags = 0;
4990	#endif
4991
4992	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4993	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4994	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4995	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4996	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4997	aParam[5] = 0;
4998
4999	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5000	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5001	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5002	#endif
5003	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5004
5005	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5006	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5007	Assert(pCtx->dr[4] == 10);
5008	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5009	#endif
5010
5011	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5012	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5013	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5014	pVCpu->hm.s.vmx.HCPhysVmcs));
5015	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5016	pCache->TestOut.HCPhysVmcs));
5017	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5018	pCache->TestOut.pCache));
5019	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5020	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5021	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5022	pCache->TestOut.pCtx));
5023	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5024	#endif
5025	return rc;
5026	}
5027
5028
5029	/**
5030	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5031	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5032	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5033	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5034	*
5035	* @returns VBox status code.
5036	* @param pVM Pointer to the VM.
5037	* @param pVCpu Pointer to the VMCPU.
5038	*/
5039	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5040	{
5041	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5042	{ \
5043	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5044	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5045	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5046	++cReadFields; \
5047	}
5048
5049	AssertPtr(pVM);
5050	AssertPtr(pVCpu);
5051	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5052	uint32_t cReadFields = 0;
5053
5054	/*
5055	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5056	* and serve to indicate exceptions to the rules.
5057	*/
5058
5059	/* Guest-natural selector base fields. */
5060	#if 0
5061	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5062	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5063	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5064	#endif
5065	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5066	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5067	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5068	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5069	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5070	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5071	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5072	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5073	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5074	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5075	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5076	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5077	#if 0
5078	/* Unused natural width guest-state fields. */
5079	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5080	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5081	#endif
5082	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5083	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5084
5085	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5086	#if 0
5087	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5088	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5089	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5090	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5091	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5092	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5093	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5094	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5095	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5096	#endif
5097
5098	/* Natural width guest-state fields. */
5099	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5100	#if 0
5101	/* Currently unused field. */
5102	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5103	#endif
5104
5105	if (pVM->hm.s.fNestedPaging)
5106	{
5107	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5108	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5109	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5110	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5111	}
5112	else
5113	{
5114	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5115	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5116	}
5117
5118	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5119	return VINF_SUCCESS;
5120	}
5121
5122
5123	/**
5124	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5125	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5126	* darwin, running 64-bit guests).
5127	*
5128	* @returns VBox status code.
5129	* @param pVCpu Pointer to the VMCPU.
5130	* @param idxField The VMCS field encoding.
5131	* @param u64Val 16, 32 or 64-bit value.
5132	*/
5133	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5134	{
5135	int rc;
5136	switch (idxField)
5137	{
5138	/*
5139	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5140	*/
5141	/* 64-bit Control fields. */
5142	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5143	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5144	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5145	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5146	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5147	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5148	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5149	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5150	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5151	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5152	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5153	case VMX_VMCS64_CTRL_EPTP_FULL:
5154	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5155	/* 64-bit Guest-state fields. */
5156	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5157	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5158	case VMX_VMCS64_GUEST_PAT_FULL:
5159	case VMX_VMCS64_GUEST_EFER_FULL:
5160	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5161	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5162	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5163	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5164	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5165	/* 64-bit Host-state fields. */
5166	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5167	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5168	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5169	{
5170	rc = VMXWriteVmcs32(idxField, u64Val);
5171	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5172	break;
5173	}
5174
5175	/*
5176	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5177	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5178	*/
5179	/* Natural-width Guest-state fields. */
5180	case VMX_VMCS_GUEST_CR3:
5181	case VMX_VMCS_GUEST_ES_BASE:
5182	case VMX_VMCS_GUEST_CS_BASE:
5183	case VMX_VMCS_GUEST_SS_BASE:
5184	case VMX_VMCS_GUEST_DS_BASE:
5185	case VMX_VMCS_GUEST_FS_BASE:
5186	case VMX_VMCS_GUEST_GS_BASE:
5187	case VMX_VMCS_GUEST_LDTR_BASE:
5188	case VMX_VMCS_GUEST_TR_BASE:
5189	case VMX_VMCS_GUEST_GDTR_BASE:
5190	case VMX_VMCS_GUEST_IDTR_BASE:
5191	case VMX_VMCS_GUEST_RSP:
5192	case VMX_VMCS_GUEST_RIP:
5193	case VMX_VMCS_GUEST_SYSENTER_ESP:
5194	case VMX_VMCS_GUEST_SYSENTER_EIP:
5195	{
5196	if (!(u64Val >> 32))
5197	{
5198	/* If this field is 64-bit, VT-x will zero out the top bits. */
5199	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5200	}
5201	else
5202	{
5203	/* Assert that only the 32->64 switcher case should ever come here. */
5204	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5205	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5206	}
5207	break;
5208	}
5209
5210	default:
5211	{
5212	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5213	rc = VERR_INVALID_PARAMETER;
5214	break;
5215	}
5216	}
5217	AssertRCReturn(rc, rc);
5218	return rc;
5219	}
5220
5221
5222	/**
5223	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5224	* hosts (except darwin) for 64-bit guests.
5225	*
5226	* @param pVCpu Pointer to the VMCPU.
5227	* @param idxField The VMCS field encoding.
5228	* @param u64Val 16, 32 or 64-bit value.
5229	*/
5230	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5231	{
5232	AssertPtr(pVCpu);
5233	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5234
5235	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5236	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5237
5238	/* Make sure there are no duplicates. */
5239	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5240	{
5241	if (pCache->Write.aField[i] == idxField)
5242	{
5243	pCache->Write.aFieldVal[i] = u64Val;
5244	return VINF_SUCCESS;
5245	}
5246	}
5247
5248	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5249	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5250	pCache->Write.cValidEntries++;
5251	return VINF_SUCCESS;
5252	}
5253
5254	/* Enable later when the assembly code uses these as callbacks. */
5255	#if 0
5256	/*
5257	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5258	*
5259	* @param pVCpu Pointer to the VMCPU.
5260	* @param pCache Pointer to the VMCS cache.
5261	*
5262	* @remarks No-long-jump zone!!!
5263	*/
5264	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5265	{
5266	AssertPtr(pCache);
5267	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5268	{
5269	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5270	AssertRC(rc);
5271	}
5272	pCache->Write.cValidEntries = 0;
5273	}
5274
5275
5276	/**
5277	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5278	*
5279	* @param pVCpu Pointer to the VMCPU.
5280	* @param pCache Pointer to the VMCS cache.
5281	*
5282	* @remarks No-long-jump zone!!!
5283	*/
5284	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5285	{
5286	AssertPtr(pCache);
5287	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5288	{
5289	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5290	AssertRC(rc);
5291	}
5292	}
5293	#endif
5294	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5295
5296
5297	/**
5298	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5299	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5300	* timer.
5301	*
5302	* @returns VBox status code.
5303	* @param pVCpu Pointer to the VMCPU.
5304	*
5305	* @remarks No-long-jump zone!!!
5306	*/
5307	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5308	{
5309	int rc = VERR_INTERNAL_ERROR_5;
5310	bool fOffsettedTsc = false;
5311	PVM pVM = pVCpu->CTX_SUFF(pVM);
5312	if (pVM->hm.s.vmx.fUsePreemptTimer)
5313	{
5314	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5315
5316	/* Make sure the returned values have sane upper and lower boundaries. */
5317	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5318	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5319	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5320	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5321
5322	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5323	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5324	}
5325	else
5326	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5327
5328	if (fOffsettedTsc)
5329	{
5330	uint64_t u64CurTSC = ASMReadTSC();
5331	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5332	{
5333	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5334	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5335
5336	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5337	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5338	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5339	}
5340	else
5341	{
5342	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5343	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5344	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5345	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5346	}
5347	}
5348	else
5349	{
5350	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5351	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5352	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5353	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5354	}
5355	}
5356
5357
5358	/**
5359	* Determines if an exception is a contributory exception. Contributory
5360	* exceptions are ones which can cause double-faults. Page-fault is
5361	* intentionally not included here as it's a conditional contributory exception.
5362	*
5363	* @returns true if the exception is contributory, false otherwise.
5364	* @param uVector The exception vector.
5365	*/
5366	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5367	{
5368	switch (uVector)
5369	{
5370	case X86_XCPT_GP:
5371	case X86_XCPT_SS:
5372	case X86_XCPT_NP:
5373	case X86_XCPT_TS:
5374	case X86_XCPT_DE:
5375	return true;
5376	default:
5377	break;
5378	}
5379	return false;
5380	}
5381
5382
5383	/**
5384	* Sets an event as a pending event to be injected into the guest.
5385	*
5386	* @param pVCpu Pointer to the VMCPU.
5387	* @param u32IntInfo The VM-entry interruption-information field.
5388	* @param cbInstr The VM-entry instruction length in bytes (for software
5389	* interrupts, exceptions and privileged software
5390	* exceptions).
5391	* @param u32ErrCode The VM-entry exception error code.
5392	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5393	* page-fault.
5394	*
5395	* @remarks Statistics counter assumes this is a guest event being injected or
5396	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5397	* always incremented.
5398	*/
5399	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5400	RTGCUINTPTR GCPtrFaultAddress)
5401	{
5402	Assert(!pVCpu->hm.s.Event.fPending);
5403	pVCpu->hm.s.Event.fPending = true;
5404	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5405	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5406	pVCpu->hm.s.Event.cbInstr = cbInstr;
5407	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5408
5409	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5410	}
5411
5412
5413	/**
5414	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5415	*
5416	* @param pVCpu Pointer to the VMCPU.
5417	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5418	* out-of-sync. Make sure to update the required fields
5419	* before using them.
5420	*/
5421	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5422	{
5423	NOREF(pMixedCtx);
5424	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5425	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5426	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5427	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5428	}
5429
5430
5431	/**
5432	* Handle a condition that occurred while delivering an event through the guest
5433	* IDT.
5434	*
5435	* @returns VBox status code (informational error codes included).
5436	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5437	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5438	* continue execution of the guest which will delivery the #DF.
5439	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5440	*
5441	* @param pVCpu Pointer to the VMCPU.
5442	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5443	* out-of-sync. Make sure to update the required fields
5444	* before using them.
5445	* @param pVmxTransient Pointer to the VMX transient structure.
5446	*
5447	* @remarks No-long-jump zone!!!
5448	*/
5449	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5450	{
5451	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5452	AssertRCReturn(rc, rc);
5453	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5454	{
5455	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5456	AssertRCReturn(rc, rc);
5457
5458	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5459	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5460	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5461
5462	typedef enum
5463	{
5464	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5465	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5466	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5467	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5468	} VMXREFLECTXCPT;
5469
5470	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5471	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5472	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5473	{
5474	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5475	{
5476	enmReflect = VMXREFLECTXCPT_XCPT;
5477	#ifdef VBOX_STRICT
5478	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5479	&& uExitVector == X86_XCPT_PF)
5480	{
5481	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5482	}
5483	#endif
5484	if ( uExitVector == X86_XCPT_PF
5485	&& uIdtVector == X86_XCPT_PF)
5486	{
5487	pVmxTransient->fVectoringPF = true;
5488	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5489	}
5490	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5491	&& hmR0VmxIsContributoryXcpt(uExitVector)
5492	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5493	\|\| uIdtVector == X86_XCPT_PF))
5494	{
5495	enmReflect = VMXREFLECTXCPT_DF;
5496	}
5497	else if (uIdtVector == X86_XCPT_DF)
5498	enmReflect = VMXREFLECTXCPT_TF;
5499	}
5500	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5501	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5502	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5503	{
5504	/*
5505	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5506	* (whatever they are) as they reoccur when restarting the instruction.
5507	*/
5508	enmReflect = VMXREFLECTXCPT_XCPT;
5509	}
5510	}
5511	else
5512	{
5513	/*
5514	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5515	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5516	* original exception to the guest after handling the VM-exit.
5517	*/
5518	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5519	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5520	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5521	{
5522	enmReflect = VMXREFLECTXCPT_XCPT;
5523	}
5524	}
5525
5526	switch (enmReflect)
5527	{
5528	case VMXREFLECTXCPT_XCPT:
5529	{
5530	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5531	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5532	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5533
5534	uint32_t u32ErrCode = 0;
5535	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5536	{
5537	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5538	AssertRCReturn(rc, rc);
5539	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5540	}
5541
5542	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5543	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5544	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5545	rc = VINF_SUCCESS;
5546	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5547	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5548
5549	break;
5550	}
5551
5552	case VMXREFLECTXCPT_DF:
5553	{
5554	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5555	rc = VINF_HM_DOUBLE_FAULT;
5556	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5557	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5558
5559	break;
5560	}
5561
5562	case VMXREFLECTXCPT_TF:
5563	{
5564	rc = VINF_EM_RESET;
5565	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5566	uExitVector));
5567	break;
5568	}
5569
5570	default:
5571	Assert(rc == VINF_SUCCESS);
5572	break;
5573	}
5574	}
5575	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5576	return rc;
5577	}
5578
5579
5580	/**
5581	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5582	*
5583	* @returns VBox status code.
5584	* @param pVCpu Pointer to the VMCPU.
5585	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5586	* out-of-sync. Make sure to update the required fields
5587	* before using them.
5588	*
5589	* @remarks No-long-jump zone!!!
5590	*/
5591	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5592	{
5593	NOREF(pMixedCtx);
5594
5595	/* Since this can be called from our preemption hook it's safer to make the guest-CR0 update non-preemptible. */
5596	VMMRZCallRing3Disable(pVCpu);
5597	HM_DISABLE_PREEMPT_IF_NEEDED();
5598
5599	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5600	{
5601	uint32_t uVal = 0;
5602	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5603	AssertRCReturn(rc, rc);
5604
5605	uint32_t uShadow = 0;
5606	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5607	AssertRCReturn(rc, rc);
5608
5609	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5610	CPUMSetGuestCR0(pVCpu, uVal);
5611	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5612	}
5613
5614	HM_RESTORE_PREEMPT_IF_NEEDED();
5615	VMMRZCallRing3Enable(pVCpu);
5616
5617	return VINF_SUCCESS;
5618	}
5619
5620
5621	/**
5622	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5623	*
5624	* @returns VBox status code.
5625	* @param pVCpu Pointer to the VMCPU.
5626	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5627	* out-of-sync. Make sure to update the required fields
5628	* before using them.
5629	*
5630	* @remarks No-long-jump zone!!!
5631	*/
5632	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5633	{
5634	NOREF(pMixedCtx);
5635
5636	int rc = VINF_SUCCESS;
5637	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5638	{
5639	uint32_t uVal = 0;
5640	uint32_t uShadow = 0;
5641	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5642	AssertRCReturn(rc, rc);
5643	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5644	AssertRCReturn(rc, rc);
5645
5646	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5647	CPUMSetGuestCR4(pVCpu, uVal);
5648	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5649	}
5650	return rc;
5651	}
5652
5653
5654	/**
5655	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5656	*
5657	* @returns VBox status code.
5658	* @param pVCpu Pointer to the VMCPU.
5659	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5660	* out-of-sync. Make sure to update the required fields
5661	* before using them.
5662	*
5663	* @remarks No-long-jump zone!!!
5664	*/
5665	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5666	{
5667	int rc = VINF_SUCCESS;
5668	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5669	{
5670	uint64_t u64Val = 0;
5671	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5672	AssertRCReturn(rc, rc);
5673
5674	pMixedCtx->rip = u64Val;
5675	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5676	}
5677	return rc;
5678	}
5679
5680
5681	/**
5682	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5683	*
5684	* @returns VBox status code.
5685	* @param pVCpu Pointer to the VMCPU.
5686	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5687	* out-of-sync. Make sure to update the required fields
5688	* before using them.
5689	*
5690	* @remarks No-long-jump zone!!!
5691	*/
5692	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5693	{
5694	int rc = VINF_SUCCESS;
5695	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5696	{
5697	uint64_t u64Val = 0;
5698	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5699	AssertRCReturn(rc, rc);
5700
5701	pMixedCtx->rsp = u64Val;
5702	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5703	}
5704	return rc;
5705	}
5706
5707
5708	/**
5709	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5710	*
5711	* @returns VBox status code.
5712	* @param pVCpu Pointer to the VMCPU.
5713	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5714	* out-of-sync. Make sure to update the required fields
5715	* before using them.
5716	*
5717	* @remarks No-long-jump zone!!!
5718	*/
5719	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5720	{
5721	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5722	{
5723	uint32_t uVal = 0;
5724	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5725	AssertRCReturn(rc, rc);
5726
5727	pMixedCtx->eflags.u32 = uVal;
5728	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5729	{
5730	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5731	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5732
5733	pMixedCtx->eflags.Bits.u1VM = 0;
5734	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5735	}
5736
5737	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5738	}
5739	return VINF_SUCCESS;
5740	}
5741
5742
5743	/**
5744	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5745	* guest-CPU context.
5746	*/
5747	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5748	{
5749	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5750	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5751	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5752	return rc;
5753	}
5754
5755
5756	/**
5757	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5758	* from the guest-state area in the VMCS.
5759	*
5760	* @param pVCpu Pointer to the VMCPU.
5761	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5762	* out-of-sync. Make sure to update the required fields
5763	* before using them.
5764	*
5765	* @remarks No-long-jump zone!!!
5766	*/
5767	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5768	{
5769	uint32_t uIntrState = 0;
5770	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5771	AssertRC(rc);
5772
5773	if (!uIntrState)
5774	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5775	else
5776	{
5777	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5778	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5779	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5780	AssertRC(rc);
5781	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5782	AssertRC(rc);
5783
5784	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5785	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5786	}
5787	}
5788
5789
5790	/**
5791	* Saves the guest's activity state.
5792	*
5793	* @returns VBox status code.
5794	* @param pVCpu Pointer to the VMCPU.
5795	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5796	* out-of-sync. Make sure to update the required fields
5797	* before using them.
5798	*
5799	* @remarks No-long-jump zone!!!
5800	*/
5801	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5802	{
5803	NOREF(pMixedCtx);
5804	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5805	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
5806	return VINF_SUCCESS;
5807	}
5808
5809
5810	/**
5811	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5812	* the current VMCS into the guest-CPU context.
5813	*
5814	* @returns VBox status code.
5815	* @param pVCpu Pointer to the VMCPU.
5816	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5817	* out-of-sync. Make sure to update the required fields
5818	* before using them.
5819	*
5820	* @remarks No-long-jump zone!!!
5821	*/
5822	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5823	{
5824	int rc = VINF_SUCCESS;
5825	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5826	{
5827	uint32_t u32Val = 0;
5828	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5829	pMixedCtx->SysEnter.cs = u32Val;
5830	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
5831	}
5832
5833	uint64_t u64Val = 0;
5834	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5835	{
5836	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5837	pMixedCtx->SysEnter.eip = u64Val;
5838	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
5839	}
5840	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5841	{
5842	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5843	pMixedCtx->SysEnter.esp = u64Val;
5844	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
5845	}
5846	return rc;
5847	}
5848
5849
5850	/**
5851	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5852	* the CPU back into the guest-CPU context.
5853	*
5854	* @returns VBox status code.
5855	* @param pVCpu Pointer to the VMCPU.
5856	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5857	* out-of-sync. Make sure to update the required fields
5858	* before using them.
5859	*
5860	* @remarks No-long-jump zone!!!
5861	*/
5862	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5863	{
5864	#if HC_ARCH_BITS == 64
5865	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5866	{
5867	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
5868	VMMRZCallRing3Disable(pVCpu);
5869	HM_DISABLE_PREEMPT_IF_NEEDED();
5870
5871	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5872	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
5873	{
5874	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5875	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5876	}
5877
5878	HM_RESTORE_PREEMPT_IF_NEEDED();
5879	VMMRZCallRing3Enable(pVCpu);
5880	}
5881	else
5882	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5883	#else
5884	NOREF(pMixedCtx);
5885	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5886	#endif
5887
5888	return VINF_SUCCESS;
5889	}
5890
5891
5892	/**
5893	* Saves the auto load/store'd guest MSRs from the current VMCS into
5894	* the guest-CPU context.
5895	*
5896	* @returns VBox status code.
5897	* @param pVCpu Pointer to the VMCPU.
5898	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5899	* out-of-sync. Make sure to update the required fields
5900	* before using them.
5901	*
5902	* @remarks No-long-jump zone!!!
5903	*/
5904	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5905	{
5906	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
5907	return VINF_SUCCESS;
5908
5909	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5910	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5911	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5912	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5913	{
5914	switch (pMsr->u32Msr)
5915	{
5916	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
5917	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5918	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5919	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5920	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5921	default:
5922	{
5923	AssertFailed();
5924	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5925	}
5926	}
5927	}
5928
5929	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
5930	return VINF_SUCCESS;
5931	}
5932
5933
5934	/**
5935	* Saves the guest control registers from the current VMCS into the guest-CPU
5936	* context.
5937	*
5938	* @returns VBox status code.
5939	* @param pVCpu Pointer to the VMCPU.
5940	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5941	* out-of-sync. Make sure to update the required fields
5942	* before using them.
5943	*
5944	* @remarks No-long-jump zone!!!
5945	*/
5946	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5947	{
5948	/* Guest CR0. Guest FPU. */
5949	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5950	AssertRCReturn(rc, rc);
5951
5952	/* Guest CR4. */
5953	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5954	AssertRCReturn(rc, rc);
5955
5956	/* Guest CR2 - updated always during the world-switch or in #PF. */
5957	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5958	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
5959	{
5960	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
5961	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
5962
5963	PVM pVM = pVCpu->CTX_SUFF(pVM);
5964	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5965	\|\| ( pVM->hm.s.fNestedPaging
5966	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5967	{
5968	uint64_t u64Val = 0;
5969	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5970	if (pMixedCtx->cr3 != u64Val)
5971	{
5972	CPUMSetGuestCR3(pVCpu, u64Val);
5973	if (VMMRZCallRing3IsEnabled(pVCpu))
5974	{
5975	PGMUpdateCR3(pVCpu, u64Val);
5976	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5977	}
5978	else
5979	{
5980	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5981	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5982	}
5983	}
5984
5985	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5986	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5987	{
5988	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5989	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5990	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5991	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5992
5993	if (VMMRZCallRing3IsEnabled(pVCpu))
5994	{
5995	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5996	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5997	}
5998	else
5999	{
6000	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6001	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6002	}
6003	}
6004	}
6005
6006	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6007	}
6008
6009	/*
6010	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6011	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6012	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6013	*
6014	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6015	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6016	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6017	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6018	*
6019	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6020	*/
6021	if (VMMRZCallRing3IsEnabled(pVCpu))
6022	{
6023	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6024	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6025
6026	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6027	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6028
6029	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6030	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6031	}
6032
6033	return rc;
6034	}
6035
6036
6037	/**
6038	* Reads a guest segment register from the current VMCS into the guest-CPU
6039	* context.
6040	*
6041	* @returns VBox status code.
6042	* @param pVCpu Pointer to the VMCPU.
6043	* @param idxSel Index of the selector in the VMCS.
6044	* @param idxLimit Index of the segment limit in the VMCS.
6045	* @param idxBase Index of the segment base in the VMCS.
6046	* @param idxAccess Index of the access rights of the segment in the VMCS.
6047	* @param pSelReg Pointer to the segment selector.
6048	*
6049	* @remarks No-long-jump zone!!!
6050	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6051	* macro as that takes care of whether to read from the VMCS cache or
6052	* not.
6053	*/
6054	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6055	PCPUMSELREG pSelReg)
6056	{
6057	NOREF(pVCpu);
6058
6059	uint32_t u32Val = 0;
6060	int rc = VMXReadVmcs32(idxSel, &u32Val);
6061	AssertRCReturn(rc, rc);
6062	pSelReg->Sel = (uint16_t)u32Val;
6063	pSelReg->ValidSel = (uint16_t)u32Val;
6064	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6065
6066	rc = VMXReadVmcs32(idxLimit, &u32Val);
6067	AssertRCReturn(rc, rc);
6068	pSelReg->u32Limit = u32Val;
6069
6070	uint64_t u64Val = 0;
6071	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6072	AssertRCReturn(rc, rc);
6073	pSelReg->u64Base = u64Val;
6074
6075	rc = VMXReadVmcs32(idxAccess, &u32Val);
6076	AssertRCReturn(rc, rc);
6077	pSelReg->Attr.u = u32Val;
6078
6079	/*
6080	* If VT-x marks the segment as unusable, most other bits remain undefined:
6081	* - For CS the L, D and G bits have meaning.
6082	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6083	* - For the remaining data segments no bits are defined.
6084	*
6085	* The present bit and the unusable bit has been observed to be set at the
6086	* same time (the selector was supposed to invalid as we started executing
6087	* a V8086 interrupt in ring-0).
6088	*
6089	* What should be important for the rest of the VBox code, is that the P bit is
6090	* cleared. Some of the other VBox code recognizes the unusable bit, but
6091	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6092	* safe side here, we'll strip off P and other bits we don't care about. If
6093	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6094	*
6095	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6096	*/
6097	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6098	{
6099	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6100
6101	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6102	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6103	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6104
6105	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6106	#ifdef DEBUG_bird
6107	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6108	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6109	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6110	#endif
6111	}
6112	return VINF_SUCCESS;
6113	}
6114
6115
6116	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6117	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6118	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6119	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6120	#else
6121	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6122	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6123	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6124	#endif
6125
6126
6127	/**
6128	* Saves the guest segment registers from the current VMCS into the guest-CPU
6129	* context.
6130	*
6131	* @returns VBox status code.
6132	* @param pVCpu Pointer to the VMCPU.
6133	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6134	* out-of-sync. Make sure to update the required fields
6135	* before using them.
6136	*
6137	* @remarks No-long-jump zone!!!
6138	*/
6139	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6140	{
6141	/* Guest segment registers. */
6142	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6143	{
6144	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6145	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6146	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6147	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6148	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6149	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6150	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6151
6152	/* Restore segment attributes for real-on-v86 mode hack. */
6153	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6154	{
6155	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6156	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6157	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6158	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6159	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6160	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6161	}
6162	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6163	}
6164
6165	return VINF_SUCCESS;
6166	}
6167
6168
6169	/**
6170	* Saves the guest descriptor table registers and task register from the current
6171	* VMCS into the guest-CPU context.
6172	*
6173	* @returns VBox status code.
6174	* @param pVCpu Pointer to the VMCPU.
6175	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6176	* out-of-sync. Make sure to update the required fields
6177	* before using them.
6178	*
6179	* @remarks No-long-jump zone!!!
6180	*/
6181	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6182	{
6183	int rc = VINF_SUCCESS;
6184
6185	/* Guest LDTR. */
6186	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6187	{
6188	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6189	AssertRCReturn(rc, rc);
6190	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6191	}
6192
6193	/* Guest GDTR. */
6194	uint64_t u64Val = 0;
6195	uint32_t u32Val = 0;
6196	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6197	{
6198	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6199	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6200	pMixedCtx->gdtr.pGdt = u64Val;
6201	pMixedCtx->gdtr.cbGdt = u32Val;
6202	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6203	}
6204
6205	/* Guest IDTR. */
6206	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6207	{
6208	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6209	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6210	pMixedCtx->idtr.pIdt = u64Val;
6211	pMixedCtx->idtr.cbIdt = u32Val;
6212	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6213	}
6214
6215	/* Guest TR. */
6216	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6217	{
6218	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6219	AssertRCReturn(rc, rc);
6220
6221	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6222	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6223	{
6224	rc = VMXLOCAL_READ_SEG(TR, tr);
6225	AssertRCReturn(rc, rc);
6226	}
6227	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6228	}
6229	return rc;
6230	}
6231
6232	#undef VMXLOCAL_READ_SEG
6233
6234
6235	/**
6236	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6237	* context.
6238	*
6239	* @returns VBox status code.
6240	* @param pVCpu Pointer to the VMCPU.
6241	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6242	* out-of-sync. Make sure to update the required fields
6243	* before using them.
6244	*
6245	* @remarks No-long-jump zone!!!
6246	*/
6247	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6248	{
6249	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6250	{
6251	if (!pVCpu->hm.s.fUsingHyperDR7)
6252	{
6253	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6254	uint32_t u32Val;
6255	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6256	pMixedCtx->dr[7] = u32Val;
6257	}
6258
6259	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6260	}
6261	return VINF_SUCCESS;
6262	}
6263
6264
6265	/**
6266	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6267	*
6268	* @returns VBox status code.
6269	* @param pVCpu Pointer to the VMCPU.
6270	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6271	* out-of-sync. Make sure to update the required fields
6272	* before using them.
6273	*
6274	* @remarks No-long-jump zone!!!
6275	*/
6276	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6277	{
6278	NOREF(pMixedCtx);
6279
6280	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6281	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6282	return VINF_SUCCESS;
6283	}
6284
6285
6286	/**
6287	* Saves the entire guest state from the currently active VMCS into the
6288	* guest-CPU context. This essentially VMREADs all guest-data.
6289	*
6290	* @returns VBox status code.
6291	* @param pVCpu Pointer to the VMCPU.
6292	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6293	* out-of-sync. Make sure to update the required fields
6294	* before using them.
6295	*/
6296	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6297	{
6298	Assert(pVCpu);
6299	Assert(pMixedCtx);
6300
6301	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6302	return VINF_SUCCESS;
6303
6304	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6305	again on the ring-3 callback path, there is no real need to. */
6306	if (VMMRZCallRing3IsEnabled(pVCpu))
6307	VMMR0LogFlushDisable(pVCpu);
6308	else
6309	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6310	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6311
6312	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6313	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6314
6315	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6317
6318	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6320
6321	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6323
6324	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6325	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6326
6327	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6328	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6329
6330	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6331	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6332
6333	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6334	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6335
6336	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6337	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6338
6339	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6340	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6341
6342	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6343	("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6344
6345	if (VMMRZCallRing3IsEnabled(pVCpu))
6346	VMMR0LogFlushEnable(pVCpu);
6347
6348	return rc;
6349	}
6350
6351
6352	/**
6353	* Check per-VM and per-VCPU force flag actions that require us to go back to
6354	* ring-3 for one reason or another.
6355	*
6356	* @returns VBox status code (information status code included).
6357	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6358	* ring-3.
6359	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6360	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6361	* interrupts)
6362	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6363	* all EMTs to be in ring-3.
6364	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6365	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6366	* to the EM loop.
6367	*
6368	* @param pVM Pointer to the VM.
6369	* @param pVCpu Pointer to the VMCPU.
6370	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6371	* out-of-sync. Make sure to update the required fields
6372	* before using them.
6373	*/
6374	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6375	{
6376	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6377
6378	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6379	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6380	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6381	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6382	{
6383	/* We need the control registers now, make sure the guest-CPU context is updated. */
6384	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6385	AssertRCReturn(rc3, rc3);
6386
6387	/* Pending HM CR3 sync. */
6388	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6389	{
6390	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6391	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6392	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6393	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6394	}
6395
6396	/* Pending HM PAE PDPEs. */
6397	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6398	{
6399	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6400	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6401	}
6402
6403	/* Pending PGM C3 sync. */
6404	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6405	{
6406	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6407	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6408	if (rc2 != VINF_SUCCESS)
6409	{
6410	AssertRC(rc2);
6411	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6412	return rc2;
6413	}
6414	}
6415
6416	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6417	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6418	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6419	{
6420	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6421	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6422	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6423	return rc2;
6424	}
6425
6426	/* Pending VM request packets, such as hardware interrupts. */
6427	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6428	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6429	{
6430	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6431	return VINF_EM_PENDING_REQUEST;
6432	}
6433
6434	/* Pending PGM pool flushes. */
6435	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6436	{
6437	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6438	return VINF_PGM_POOL_FLUSH_PENDING;
6439	}
6440
6441	/* Pending DMA requests. */
6442	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6443	{
6444	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6445	return VINF_EM_RAW_TO_R3;
6446	}
6447	}
6448
6449	return VINF_SUCCESS;
6450	}
6451
6452
6453	/**
6454	* Converts any TRPM trap into a pending HM event. This is typically used when
6455	* entering from ring-3 (not longjmp returns).
6456	*
6457	* @param pVCpu Pointer to the VMCPU.
6458	*/
6459	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6460	{
6461	Assert(TRPMHasTrap(pVCpu));
6462	Assert(!pVCpu->hm.s.Event.fPending);
6463
6464	uint8_t uVector;
6465	TRPMEVENT enmTrpmEvent;
6466	RTGCUINT uErrCode;
6467	RTGCUINTPTR GCPtrFaultAddress;
6468	uint8_t cbInstr;
6469
6470	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6471	AssertRC(rc);
6472
6473	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6474	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6475	if (enmTrpmEvent == TRPM_TRAP)
6476	{
6477	switch (uVector)
6478	{
6479	case X86_XCPT_BP:
6480	case X86_XCPT_OF:
6481	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6482	break;
6483
6484	case X86_XCPT_PF:
6485	case X86_XCPT_DF:
6486	case X86_XCPT_TS:
6487	case X86_XCPT_NP:
6488	case X86_XCPT_SS:
6489	case X86_XCPT_GP:
6490	case X86_XCPT_AC:
6491	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6492	/* no break! */
6493	default:
6494	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6495	break;
6496	}
6497	}
6498	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6499	{
6500	if (uVector == X86_XCPT_NMI)
6501	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6502	else
6503	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6504	}
6505	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6506	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6507	else
6508	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6509
6510	rc = TRPMResetTrap(pVCpu);
6511	AssertRC(rc);
6512	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6513	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6514
6515	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6516	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6517	}
6518
6519
6520	/**
6521	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6522	* VT-x to execute any instruction.
6523	*
6524	* @param pvCpu Pointer to the VMCPU.
6525	*/
6526	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6527	{
6528	Assert(pVCpu->hm.s.Event.fPending);
6529
6530	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6531	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6532	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6533	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6534
6535	/* If a trap was already pending, we did something wrong! */
6536	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6537
6538	TRPMEVENT enmTrapType;
6539	switch (uVectorType)
6540	{
6541	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6542	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6543	enmTrapType = TRPM_HARDWARE_INT;
6544	break;
6545
6546	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6547	enmTrapType = TRPM_SOFTWARE_INT;
6548	break;
6549
6550	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6551	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6552	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6553	enmTrapType = TRPM_TRAP;
6554	break;
6555
6556	default:
6557	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6558	enmTrapType = TRPM_32BIT_HACK;
6559	break;
6560	}
6561
6562	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6563
6564	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6565	AssertRC(rc);
6566
6567	if (fErrorCodeValid)
6568	TRPMSetErrorCode(pVCpu, uErrorCode);
6569
6570	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6571	&& uVector == X86_XCPT_PF)
6572	{
6573	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6574	}
6575	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6576	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6577	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6578	{
6579	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6580	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6581	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6582	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6583	}
6584	pVCpu->hm.s.Event.fPending = false;
6585	}
6586
6587
6588	/**
6589	* Does the necessary state syncing before returning to ring-3 for any reason
6590	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6591	*
6592	* @returns VBox status code.
6593	* @param pVM Pointer to the VM.
6594	* @param pVCpu Pointer to the VMCPU.
6595	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6596	* be out-of-sync. Make sure to update the required
6597	* fields before using them.
6598	* @param fSaveGuestState Whether to save the guest state or not.
6599	*
6600	* @remarks No-long-jmp zone!!!
6601	*/
6602	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6603	{
6604	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6605	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6606
6607	RTCPUID idCpu = RTMpCpuId();
6608	Log4Func(("HostCpuId=%u\n", idCpu));
6609
6610	/*
6611	* !!! IMPORTANT !!!
6612	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6613	*/
6614
6615	/* Save the guest state if necessary. */
6616	if ( fSaveGuestState
6617	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6618	{
6619	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6620	AssertRCReturn(rc, rc);
6621	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6622	}
6623
6624	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6625	if (CPUMIsGuestFPUStateActive(pVCpu))
6626	{
6627	/* We shouldn't reload CR0 without saving it first. */
6628	if (!fSaveGuestState)
6629	{
6630	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6631	AssertRCReturn(rc, rc);
6632	}
6633	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6634	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6635	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6636	}
6637
6638	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6639	#ifdef VBOX_STRICT
6640	if (CPUMIsHyperDebugStateActive(pVCpu))
6641	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6642	#endif
6643	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6644	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6645	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6646	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6647
6648	#if HC_ARCH_BITS == 64
6649	/* Restore host-state bits that VT-x only restores partially. */
6650	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6651	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6652	{
6653	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6654	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6655	}
6656	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6657	#endif
6658
6659	#if HC_ARCH_BITS == 64
6660	/* Restore the host MSRs as we're leaving VT-x context. */
6661	if ( pVM->hm.s.fAllow64BitGuests
6662	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6663	{
6664	/* We shouldn't reload the guest MSRs without saving it first. */
6665	if (!fSaveGuestState)
6666	{
6667	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6668	AssertRCReturn(rc, rc);
6669	}
6670	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6671	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6672	Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6673	}
6674	#endif
6675
6676	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6677	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6678	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6679	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6680	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6681	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6682	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6683	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6684
6685	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6686
6687	/** @todo This partially defeats the purpose of having preemption hooks.
6688	* The problem is, deregistering the hooks should be moved to a place that
6689	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6690	* context.
6691	*/
6692	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6693	{
6694	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6695	AssertRCReturn(rc, rc);
6696
6697	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6698	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6699	}
6700	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6701	NOREF(idCpu);
6702
6703	return VINF_SUCCESS;
6704	}
6705
6706
6707	/**
6708	* Leaves the VT-x session.
6709	*
6710	* @returns VBox status code.
6711	* @param pVM Pointer to the VM.
6712	* @param pVCpu Pointer to the VMCPU.
6713	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6714	* out-of-sync. Make sure to update the required fields
6715	* before using them.
6716	*
6717	* @remarks No-long-jmp zone!!!
6718	*/
6719	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6720	{
6721	HM_DISABLE_PREEMPT_IF_NEEDED();
6722	HMVMX_ASSERT_CPU_SAFE();
6723	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6724	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6725
6726	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6727	and done this from the VMXR0ThreadCtxCallback(). */
6728	if (!pVCpu->hm.s.fLeaveDone)
6729	{
6730	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6731	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6732	pVCpu->hm.s.fLeaveDone = true;
6733	}
6734
6735	/*
6736	* !!! IMPORTANT !!!
6737	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6738	*/
6739
6740	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6741	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6742	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6743	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6744	VMMR0ThreadCtxHooksDeregister(pVCpu);
6745
6746	/* Leave HM context. This takes care of local init (term). */
6747	int rc = HMR0LeaveCpu(pVCpu);
6748
6749	HM_RESTORE_PREEMPT_IF_NEEDED();
6750
6751	return rc;
6752	}
6753
6754
6755	/**
6756	* Does the necessary state syncing before doing a longjmp to ring-3.
6757	*
6758	* @returns VBox status code.
6759	* @param pVM Pointer to the VM.
6760	* @param pVCpu Pointer to the VMCPU.
6761	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6762	* out-of-sync. Make sure to update the required fields
6763	* before using them.
6764	*
6765	* @remarks No-long-jmp zone!!!
6766	*/
6767	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6768	{
6769	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6770	}
6771
6772
6773	/**
6774	* Take necessary actions before going back to ring-3.
6775	*
6776	* An action requires us to go back to ring-3. This function does the necessary
6777	* steps before we can safely return to ring-3. This is not the same as longjmps
6778	* to ring-3, this is voluntary and prepares the guest so it may continue
6779	* executing outside HM (recompiler/IEM).
6780	*
6781	* @returns VBox status code.
6782	* @param pVM Pointer to the VM.
6783	* @param pVCpu Pointer to the VMCPU.
6784	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6785	* out-of-sync. Make sure to update the required fields
6786	* before using them.
6787	* @param rcExit The reason for exiting to ring-3. Can be
6788	* VINF_VMM_UNKNOWN_RING3_CALL.
6789	*/
6790	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6791	{
6792	Assert(pVM);
6793	Assert(pVCpu);
6794	Assert(pMixedCtx);
6795	HMVMX_ASSERT_PREEMPT_SAFE();
6796
6797	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6798	{
6799	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6800	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6801	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6802	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6803	}
6804
6805	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6806	VMMRZCallRing3Disable(pVCpu);
6807	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6808
6809	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6810	if (pVCpu->hm.s.Event.fPending)
6811	{
6812	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6813	Assert(!pVCpu->hm.s.Event.fPending);
6814	}
6815
6816	/* Save guest state and restore host state bits. */
6817	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6818	AssertRCReturn(rc, rc);
6819	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6820
6821	/* Sync recompiler state. */
6822	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6823	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6824	\| CPUM_CHANGED_LDTR
6825	\| CPUM_CHANGED_GDTR
6826	\| CPUM_CHANGED_IDTR
6827	\| CPUM_CHANGED_TR
6828	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6829	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6830	if ( pVM->hm.s.fNestedPaging
6831	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6832	{
6833	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6834	}
6835
6836	Assert(!pVCpu->hm.s.fClearTrapFlag);
6837
6838	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6839	if (rcExit != VINF_EM_RAW_INTERRUPT)
6840	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6841
6842	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6843
6844	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6845	VMMRZCallRing3RemoveNotification(pVCpu);
6846	VMMRZCallRing3Enable(pVCpu);
6847
6848	return rc;
6849	}
6850
6851
6852	/**
6853	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6854	* longjump to ring-3 and possibly get preempted.
6855	*
6856	* @returns VBox status code.
6857	* @param pVCpu Pointer to the VMCPU.
6858	* @param enmOperation The operation causing the ring-3 longjump.
6859	* @param pvUser Opaque pointer to the guest-CPU context. The data
6860	* may be out-of-sync. Make sure to update the required
6861	* fields before using them.
6862	*/
6863	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6864	{
6865	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6866	{
6867	/*
6868	* !!! IMPORTANT !!!
6869	* If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
6870	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
6871	*/
6872	VMMRZCallRing3RemoveNotification(pVCpu);
6873	VMMRZCallRing3Disable(pVCpu);
6874	HM_DISABLE_PREEMPT_IF_NEEDED();
6875
6876	PVM pVM = pVCpu->CTX_SUFF(pVM);
6877	if (CPUMIsGuestFPUStateActive(pVCpu))
6878	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6879
6880	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6881
6882	#if HC_ARCH_BITS == 64
6883	/* Restore host-state bits that VT-x only restores partially. */
6884	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6885	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6886	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6887	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6888
6889	/* Restore the host MSRs as we're leaving VT-x context. */
6890	if ( pVM->hm.s.fAllow64BitGuests
6891	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6892	{
6893	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6894	}
6895	#endif
6896	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6897	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6898	{
6899	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6900	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6901	}
6902
6903	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6904	VMMR0ThreadCtxHooksDeregister(pVCpu);
6905
6906	HMR0LeaveCpu(pVCpu);
6907	HM_RESTORE_PREEMPT_IF_NEEDED();
6908	return VINF_SUCCESS;
6909	}
6910
6911	Assert(pVCpu);
6912	Assert(pvUser);
6913	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6914	HMVMX_ASSERT_PREEMPT_SAFE();
6915
6916	VMMRZCallRing3Disable(pVCpu);
6917	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6918
6919	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6920	enmOperation));
6921
6922	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6923	AssertRCReturn(rc, rc);
6924
6925	VMMRZCallRing3Enable(pVCpu);
6926	return VINF_SUCCESS;
6927	}
6928
6929
6930	/**
6931	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6932	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6933	*
6934	* @param pVCpu Pointer to the VMCPU.
6935	*/
6936	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6937	{
6938	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6939	{
6940	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6941	{
6942	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6943	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6944	AssertRC(rc);
6945	}
6946	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6947	}
6948
6949
6950	/**
6951	* Evaluates the event to be delivered to the guest and sets it as the pending
6952	* event.
6953	*
6954	* @param pVCpu Pointer to the VMCPU.
6955	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6956	* out-of-sync. Make sure to update the required fields
6957	* before using them.
6958	*/
6959	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6960	{
6961	Assert(!pVCpu->hm.s.Event.fPending);
6962
6963	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6964	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6965	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6966	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6967
6968	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
6969	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6970	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6971	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6972	Assert(!TRPMHasTrap(pVCpu));
6973
6974	/** @todo SMI. SMIs take priority over NMIs. */
6975	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6976	{
6977	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6978	if ( !fBlockMovSS
6979	&& !fBlockSti)
6980	{
6981	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6982	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6983	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6984	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6985
6986	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6987	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6988	}
6989	else
6990	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6991	}
6992	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6993	&& !pVCpu->hm.s.fSingleInstruction)
6994	{
6995	/*
6996	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6997	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6998	* evaluated here and not set as pending, solely based on the force-flags.
6999	*/
7000	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7001	AssertRC(rc);
7002	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7003	if ( !fBlockInt
7004	&& !fBlockSti
7005	&& !fBlockMovSS)
7006	{
7007	uint8_t u8Interrupt;
7008	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7009	if (RT_SUCCESS(rc))
7010	{
7011	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7012	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7013	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7014
7015	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7016	}
7017	else
7018	{
7019	/** @todo Does this actually happen? If not turn it into an assertion. */
7020	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7021	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7022	}
7023	}
7024	else
7025	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7026	}
7027	}
7028
7029
7030	/**
7031	* Sets a pending-debug exception to be delivered to the guest if the guest is
7032	* single-stepping.
7033	*
7034	* @param pVCpu Pointer to the VMCPU.
7035	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7036	* out-of-sync. Make sure to update the required fields
7037	* before using them.
7038	*/
7039	DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7040	{
7041	HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
7042	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7043	{
7044	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7045	AssertRC(rc);
7046	}
7047	}
7048
7049
7050	/**
7051	* Injects any pending events into the guest if the guest is in a state to
7052	* receive them.
7053	*
7054	* @returns VBox status code (informational status codes included).
7055	* @param pVCpu Pointer to the VMCPU.
7056	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7057	* out-of-sync. Make sure to update the required fields
7058	* before using them.
7059	*/
7060	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7061	{
7062	HMVMX_ASSERT_PREEMPT_SAFE();
7063	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7064
7065	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7066	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7067	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7068	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7069
7070	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7071	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7072	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7073	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7074	Assert(!TRPMHasTrap(pVCpu));
7075
7076	int rc = VINF_SUCCESS;
7077	if (pVCpu->hm.s.Event.fPending)
7078	{
7079	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
7080	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7081	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7082	{
7083	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7084	AssertRCReturn(rc, rc);
7085	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7086	Assert(!fBlockInt);
7087	Assert(!fBlockSti);
7088	Assert(!fBlockMovSS);
7089	}
7090	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7091	{
7092	Assert(!fBlockSti);
7093	Assert(!fBlockMovSS);
7094	}
7095	#endif
7096	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
7097	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7098	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7099	AssertRCReturn(rc, rc);
7100
7101	/* Update the interruptibility-state as it could have been changed by
7102	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7103	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7104	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7105
7106	#ifdef VBOX_WITH_STATISTICS
7107	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7108	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7109	else
7110	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7111	#endif
7112	}
7113
7114	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7115	if ( fBlockSti
7116	\|\| fBlockMovSS)
7117	{
7118	if ( !pVCpu->hm.s.fSingleInstruction
7119	&& !DBGFIsStepping(pVCpu))
7120	{
7121	/*
7122	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7123	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7124	* See Intel spec. 27.3.4 "Saving Non-Register State".
7125	*/
7126	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7127	AssertRCReturn(rc2, rc2);
7128	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7129	}
7130	else if (pMixedCtx->eflags.Bits.u1TF)
7131	{
7132	/*
7133	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7134	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7135	*/
7136	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7137	uIntrState = 0;
7138	}
7139	}
7140
7141	/*
7142	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7143	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7144	*/
7145	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7146	AssertRC(rc2);
7147
7148	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7149	NOREF(fBlockMovSS); NOREF(fBlockSti);
7150	return rc;
7151	}
7152
7153
7154	/**
7155	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7156	*
7157	* @param pVCpu Pointer to the VMCPU.
7158	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7159	* out-of-sync. Make sure to update the required fields
7160	* before using them.
7161	*/
7162	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7163	{
7164	NOREF(pMixedCtx);
7165	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7166	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7167	}
7168
7169
7170	/**
7171	* Injects a double-fault (#DF) exception into the VM.
7172	*
7173	* @returns VBox status code (informational status code included).
7174	* @param pVCpu Pointer to the VMCPU.
7175	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7176	* out-of-sync. Make sure to update the required fields
7177	* before using them.
7178	*/
7179	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7180	{
7181	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7182	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7183	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7184	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7185	puIntrState);
7186	}
7187
7188
7189	/**
7190	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7191	*
7192	* @param pVCpu Pointer to the VMCPU.
7193	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7194	* out-of-sync. Make sure to update the required fields
7195	* before using them.
7196	*/
7197	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7198	{
7199	NOREF(pMixedCtx);
7200	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7201	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7202	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7203	}
7204
7205
7206	/**
7207	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7208	*
7209	* @param pVCpu Pointer to the VMCPU.
7210	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7211	* out-of-sync. Make sure to update the required fields
7212	* before using them.
7213	* @param cbInstr The value of RIP that is to be pushed on the guest
7214	* stack.
7215	*/
7216	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7217	{
7218	NOREF(pMixedCtx);
7219	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7220	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7221	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7222	}
7223
7224
7225	/**
7226	* Injects a general-protection (#GP) fault into the VM.
7227	*
7228	* @returns VBox status code (informational status code included).
7229	* @param pVCpu Pointer to the VMCPU.
7230	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7231	* out-of-sync. Make sure to update the required fields
7232	* before using them.
7233	* @param u32ErrorCode The error code associated with the #GP.
7234	*/
7235	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7236	uint32_t *puIntrState)
7237	{
7238	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7239	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7240	if (fErrorCodeValid)
7241	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7242	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7243	puIntrState);
7244	}
7245
7246
7247	/**
7248	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7249	*
7250	* @param pVCpu Pointer to the VMCPU.
7251	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7252	* out-of-sync. Make sure to update the required fields
7253	* before using them.
7254	* @param uVector The software interrupt vector number.
7255	* @param cbInstr The value of RIP that is to be pushed on the guest
7256	* stack.
7257	*/
7258	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7259	{
7260	NOREF(pMixedCtx);
7261	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7262	if ( uVector == X86_XCPT_BP
7263	\|\| uVector == X86_XCPT_OF)
7264	{
7265	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7266	}
7267	else
7268	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7269	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7270	}
7271
7272
7273	/**
7274	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7275	* stack.
7276	*
7277	* @returns VBox status code (information status code included).
7278	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7279	* @param pVM Pointer to the VM.
7280	* @param pMixedCtx Pointer to the guest-CPU context.
7281	* @param uValue The value to push to the guest stack.
7282	*/
7283	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7284	{
7285	/*
7286	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7287	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7288	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7289	*/
7290	if (pMixedCtx->sp == 1)
7291	return VINF_EM_RESET;
7292	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7293	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7294	AssertRCReturn(rc, rc);
7295	return rc;
7296	}
7297
7298
7299	/**
7300	* Injects an event into the guest upon VM-entry by updating the relevant fields
7301	* in the VM-entry area in the VMCS.
7302	*
7303	* @returns VBox status code (informational error codes included).
7304	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7305	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7306	*
7307	* @param pVCpu Pointer to the VMCPU.
7308	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7309	* be out-of-sync. Make sure to update the required
7310	* fields before using them.
7311	* @param u64IntInfo The VM-entry interruption-information field.
7312	* @param cbInstr The VM-entry instruction length in bytes (for
7313	* software interrupts, exceptions and privileged
7314	* software exceptions).
7315	* @param u32ErrCode The VM-entry exception error code.
7316	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7317	* @param puIntrState Pointer to the current guest interruptibility-state.
7318	* This interruptibility-state will be updated if
7319	* necessary. This cannot not be NULL.
7320	*
7321	* @remarks Requires CR0!
7322	* @remarks No-long-jump zone!!!
7323	*/
7324	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7325	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7326	{
7327	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7328	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7329	Assert(puIntrState);
7330	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7331
7332	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7333	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7334
7335	#ifdef VBOX_STRICT
7336	/* Validate the error-code-valid bit for hardware exceptions. */
7337	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7338	{
7339	switch (uVector)
7340	{
7341	case X86_XCPT_PF:
7342	case X86_XCPT_DF:
7343	case X86_XCPT_TS:
7344	case X86_XCPT_NP:
7345	case X86_XCPT_SS:
7346	case X86_XCPT_GP:
7347	case X86_XCPT_AC:
7348	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7349	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7350	/* fallthru */
7351	default:
7352	break;
7353	}
7354	}
7355	#endif
7356
7357	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7358	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7359	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7360
7361	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7362
7363	/* We require CR0 to check if the guest is in real-mode. */
7364	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7365	AssertRCReturn(rc, rc);
7366
7367	/*
7368	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7369	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7370	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7371	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7372	*/
7373	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7374	{
7375	PVM pVM = pVCpu->CTX_SUFF(pVM);
7376	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7377	{
7378	Assert(PDMVmmDevHeapIsEnabled(pVM));
7379	Assert(pVM->hm.s.vmx.pRealModeTSS);
7380
7381	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7382	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7383	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7384	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7385	AssertRCReturn(rc, rc);
7386	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7387
7388	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7389	const size_t cbIdtEntry = sizeof(X86IDTR16);
7390	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7391	{
7392	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7393	if (uVector == X86_XCPT_DF)
7394	return VINF_EM_RESET;
7395	else if (uVector == X86_XCPT_GP)
7396	{
7397	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7398	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7399	}
7400
7401	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7402	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7403	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7404	}
7405
7406	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7407	uint16_t uGuestIp = pMixedCtx->ip;
7408	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7409	{
7410	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7411	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7412	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7413	}
7414	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7415	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7416
7417	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7418	X86IDTR16 IdtEntry;
7419	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7420	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7421	AssertRCReturn(rc, rc);
7422
7423	/* Construct the stack frame for the interrupt/exception handler. */
7424	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7425	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7426	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7427	AssertRCReturn(rc, rc);
7428
7429	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7430	if (rc == VINF_SUCCESS)
7431	{
7432	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7433	pMixedCtx->rip = IdtEntry.offSel;
7434	pMixedCtx->cs.Sel = IdtEntry.uSel;
7435	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7436	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7437	&& uVector == X86_XCPT_PF)
7438	{
7439	pMixedCtx->cr2 = GCPtrFaultAddress;
7440	}
7441
7442	/* If any other guest-state bits are changed here, make sure to update
7443	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7444	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7445	\| HM_CHANGED_GUEST_RIP
7446	\| HM_CHANGED_GUEST_RFLAGS
7447	\| HM_CHANGED_GUEST_RSP);
7448
7449	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7450	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7451	{
7452	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7453	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7454	Log4(("Clearing inhibition due to STI.\n"));
7455	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7456	}
7457	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7458
7459	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7460	it, if we are returning to ring-3 before executing guest code. */
7461	pVCpu->hm.s.Event.fPending = false;
7462	}
7463	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7464	return rc;
7465	}
7466	else
7467	{
7468	/*
7469	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7470	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7471	*/
7472	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7473	}
7474	}
7475
7476	/* Validate. */
7477	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7478	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7479	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7480
7481	/* Inject. */
7482	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7483	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7484	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7485	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7486
7487	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7488	&& uVector == X86_XCPT_PF)
7489	{
7490	pMixedCtx->cr2 = GCPtrFaultAddress;
7491	}
7492
7493	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7494	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7495
7496	AssertRCReturn(rc, rc);
7497	return rc;
7498	}
7499
7500
7501	/**
7502	* Clears the interrupt-window exiting control in the VMCS and if necessary
7503	* clears the current event in the VMCS as well.
7504	*
7505	* @returns VBox status code.
7506	* @param pVCpu Pointer to the VMCPU.
7507	*
7508	* @remarks Use this function only to clear events that have not yet been
7509	* delivered to the guest but are injected in the VMCS!
7510	* @remarks No-long-jump zone!!!
7511	*/
7512	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7513	{
7514	int rc;
7515	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7516
7517	/* Clear interrupt-window exiting control. */
7518	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7519	{
7520	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7521	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7522	AssertRC(rc);
7523	}
7524
7525	if (!pVCpu->hm.s.Event.fPending)
7526	return;
7527
7528	#ifdef VBOX_STRICT
7529	uint32_t u32EntryInfo;
7530	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7531	AssertRC(rc);
7532	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7533	#endif
7534
7535	/* Clear the entry-interruption field (including the valid bit). */
7536	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7537	AssertRC(rc);
7538
7539	/* Clear the pending debug exception field. */
7540	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7541	AssertRC(rc);
7542	}
7543
7544
7545	/**
7546	* Enters the VT-x session.
7547	*
7548	* @returns VBox status code.
7549	* @param pVM Pointer to the VM.
7550	* @param pVCpu Pointer to the VMCPU.
7551	* @param pCpu Pointer to the CPU info struct.
7552	*/
7553	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7554	{
7555	AssertPtr(pVM);
7556	AssertPtr(pVCpu);
7557	Assert(pVM->hm.s.vmx.fSupported);
7558	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7559	NOREF(pCpu); NOREF(pVM);
7560
7561	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7562	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7563
7564	#ifdef VBOX_STRICT
7565	/* Make sure we're in VMX root mode. */
7566	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7567	if (!(u32HostCR4 & X86_CR4_VMXE))
7568	{
7569	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7570	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7571	}
7572	#endif
7573
7574	/*
7575	* Load the VCPU's VMCS as the current (and active) one.
7576	*/
7577	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7578	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7579	if (RT_FAILURE(rc))
7580	return rc;
7581
7582	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7583	pVCpu->hm.s.fLeaveDone = false;
7584	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7585
7586	return VINF_SUCCESS;
7587	}
7588
7589
7590	/**
7591	* The thread-context callback (only on platforms which support it).
7592	*
7593	* @param enmEvent The thread-context event.
7594	* @param pVCpu Pointer to the VMCPU.
7595	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7596	* @thread EMT(pVCpu)
7597	*/
7598	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7599	{
7600	NOREF(fGlobalInit);
7601
7602	switch (enmEvent)
7603	{
7604	case RTTHREADCTXEVENT_PREEMPTING:
7605	{
7606	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7607	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7608	VMCPU_ASSERT_EMT(pVCpu);
7609
7610	PVM pVM = pVCpu->CTX_SUFF(pVM);
7611	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7612
7613	/* No longjmps (logger flushes, locks) in this fragile context. */
7614	VMMRZCallRing3Disable(pVCpu);
7615	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7616
7617	/*
7618	* Restore host-state (FPU, debug etc.)
7619	*/
7620	if (!pVCpu->hm.s.fLeaveDone)
7621	{
7622	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7623	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7624	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7625	pVCpu->hm.s.fLeaveDone = true;
7626	}
7627
7628	/* Leave HM context, takes care of local init (term). */
7629	int rc = HMR0LeaveCpu(pVCpu);
7630	AssertRC(rc); NOREF(rc);
7631
7632	/* Restore longjmp state. */
7633	VMMRZCallRing3Enable(pVCpu);
7634	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7635	break;
7636	}
7637
7638	case RTTHREADCTXEVENT_RESUMED:
7639	{
7640	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7641	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7642	VMCPU_ASSERT_EMT(pVCpu);
7643
7644	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7645	VMMRZCallRing3Disable(pVCpu);
7646	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7647
7648	/* Initialize the bare minimum state required for HM. This takes care of
7649	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7650	int rc = HMR0EnterCpu(pVCpu);
7651	AssertRC(rc);
7652	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7653
7654	/* Load the active VMCS as the current one. */
7655	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7656	{
7657	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7658	AssertRC(rc); NOREF(rc);
7659	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7660	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7661	}
7662	pVCpu->hm.s.fLeaveDone = false;
7663
7664	/* Restore longjmp state. */
7665	VMMRZCallRing3Enable(pVCpu);
7666	break;
7667	}
7668
7669	default:
7670	break;
7671	}
7672	}
7673
7674
7675	/**
7676	* Saves the host state in the VMCS host-state.
7677	* Sets up the VM-exit MSR-load area.
7678	*
7679	* The CPU state will be loaded from these fields on every successful VM-exit.
7680	*
7681	* @returns VBox status code.
7682	* @param pVM Pointer to the VM.
7683	* @param pVCpu Pointer to the VMCPU.
7684	*
7685	* @remarks No-long-jump zone!!!
7686	*/
7687	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7688	{
7689	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7690
7691	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7692	return VINF_SUCCESS;
7693
7694	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7695	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7696
7697	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7698	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7699
7700	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7701	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7702
7703	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7704	return rc;
7705	}
7706
7707
7708	/**
7709	* Saves the host state in the VMCS host-state.
7710	*
7711	* @returns VBox status code.
7712	* @param pVM Pointer to the VM.
7713	* @param pVCpu Pointer to the VMCPU.
7714	*
7715	* @remarks No-long-jump zone!!!
7716	*/
7717	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7718	{
7719	AssertPtr(pVM);
7720	AssertPtr(pVCpu);
7721
7722	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7723
7724	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7725	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7726	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7727	return hmR0VmxSaveHostState(pVM, pVCpu);
7728	}
7729
7730
7731	/**
7732	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7733	* loaded from these fields on every successful VM-entry.
7734	*
7735	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7736	* Sets up the VM-entry controls.
7737	* Sets up the appropriate VMX non-root function to execute guest code based on
7738	* the guest CPU mode.
7739	*
7740	* @returns VBox status code.
7741	* @param pVM Pointer to the VM.
7742	* @param pVCpu Pointer to the VMCPU.
7743	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7744	* out-of-sync. Make sure to update the required fields
7745	* before using them.
7746	*
7747	* @remarks No-long-jump zone!!!
7748	*/
7749	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7750	{
7751	AssertPtr(pVM);
7752	AssertPtr(pVCpu);
7753	AssertPtr(pMixedCtx);
7754	HMVMX_ASSERT_PREEMPT_SAFE();
7755
7756	#ifdef LOG_ENABLED
7757	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7758	* probably not initialized yet? Anyway this will do for now.
7759	*
7760	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7761	* interface and disable ring-3 calls when thread-context hooks are not
7762	* available. */
7763	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7764	VMMR0LogFlushDisable(pVCpu);
7765	#endif
7766
7767	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7768
7769	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7770
7771	/* Determine real-on-v86 mode. */
7772	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7773	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7774	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7775	{
7776	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7777	}
7778
7779	/*
7780	* Load the guest-state into the VMCS.
7781	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7782	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7783	*/
7784	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7785	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7786
7787	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7788	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7789	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7790
7791	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7792	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7793	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7794
7795	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7796	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7797
7798	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7799	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7800
7801	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7802	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7803	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7804
7805	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7806	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7807
7808	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7809	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7810
7811	/*
7812	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7813	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7814	*/
7815	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7816	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7817
7818	/* Clear any unused and reserved bits. */
7819	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7820
7821	#ifdef LOG_ENABLED
7822	/* Only reenable log-flushing if the caller has it enabled. */
7823	if (!fCallerDisabledLogFlush)
7824	VMMR0LogFlushEnable(pVCpu);
7825	#endif
7826
7827	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7828	return rc;
7829	}
7830
7831
7832	/**
7833	* Loads the state shared between the host and guest into the VMCS.
7834	*
7835	* @param pVM Pointer to the VM.
7836	* @param pVCpu Pointer to the VMCPU.
7837	* @param pCtx Pointer to the guest-CPU context.
7838	*
7839	* @remarks No-long-jump zone!!!
7840	*/
7841	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7842	{
7843	NOREF(pVM);
7844
7845	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7846	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7847
7848	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7849	{
7850	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7851	AssertRC(rc);
7852	}
7853
7854	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7855	{
7856	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7857	AssertRC(rc);
7858
7859	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7860	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7861	{
7862	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7863	AssertRC(rc);
7864	}
7865	}
7866
7867	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7868	{
7869	#if HC_ARCH_BITS == 64
7870	if (pVM->hm.s.fAllow64BitGuests)
7871	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7872	#endif
7873	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7874	}
7875
7876	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7877	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7878	}
7879
7880
7881	/**
7882	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7883	*
7884	* @param pVM Pointer to the VM.
7885	* @param pVCpu Pointer to the VMCPU.
7886	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7887	* out-of-sync. Make sure to update the required fields
7888	* before using them.
7889	*/
7890	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7891	{
7892	HMVMX_ASSERT_PREEMPT_SAFE();
7893
7894	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7895	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7896	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7897	#endif
7898
7899	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7900	{
7901	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7902	AssertRC(rc);
7903	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7904	}
7905	else if (HMCPU_CF_VALUE(pVCpu))
7906	{
7907	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7908	AssertRC(rc);
7909	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7910	}
7911
7912	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7913	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7914	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7915	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7916
7917	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7918	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7919	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7920	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7921	#endif
7922	}
7923
7924
7925	/**
7926	* Does the preparations before executing guest code in VT-x.
7927	*
7928	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7929	* recompiler. We must be cautious what we do here regarding committing
7930	* guest-state information into the VMCS assuming we assuredly execute the
7931	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7932	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7933	* so that the recompiler can (and should) use them when it resumes guest
7934	* execution. Otherwise such operations must be done when we can no longer
7935	* exit to ring-3.
7936	*
7937	* @returns Strict VBox status code.
7938	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7939	* have been disabled.
7940	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7941	* double-fault into the guest.
7942	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7943	*
7944	* @param pVM Pointer to the VM.
7945	* @param pVCpu Pointer to the VMCPU.
7946	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7947	* out-of-sync. Make sure to update the required fields
7948	* before using them.
7949	* @param pVmxTransient Pointer to the VMX transient structure.
7950	*/
7951	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7952	{
7953	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7954
7955	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7956	PGMRZDynMapFlushAutoSet(pVCpu);
7957	#endif
7958
7959	/* Check force flag actions that might require us to go back to ring-3. */
7960	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7961	if (rc != VINF_SUCCESS)
7962	return rc;
7963
7964	#ifndef IEM_VERIFICATION_MODE_FULL
7965	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7966	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7967	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7968	{
7969	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7970	RTGCPHYS GCPhysApicBase;
7971	GCPhysApicBase = pMixedCtx->msrApicBase;
7972	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7973
7974	/* Unalias any existing mapping. */
7975	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7976	AssertRCReturn(rc, rc);
7977
7978	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7979	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7980	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7981	AssertRCReturn(rc, rc);
7982
7983	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7984	}
7985	#endif /* !IEM_VERIFICATION_MODE_FULL */
7986
7987	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7988	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7989
7990	/*
7991	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7992	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7993	*/
7994	if (TRPMHasTrap(pVCpu))
7995	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7996	else if (!pVCpu->hm.s.Event.fPending)
7997	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7998
7999	/*
8000	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8001	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8002	*/
8003	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8004	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8005	{
8006	Assert(rc == VINF_EM_RESET);
8007	return rc;
8008	}
8009
8010	/*
8011	* No longjmps to ring-3 from this point on!!!
8012	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8013	* This also disables flushing of the R0-logger instance (if any).
8014	*/
8015	VMMRZCallRing3Disable(pVCpu);
8016
8017	/*
8018	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8019	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8020	*
8021	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8022	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8023	*
8024	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8025	* executing guest code.
8026	*/
8027	pVmxTransient->uEflags = ASMIntDisableFlags();
8028	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8029	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8030	{
8031	hmR0VmxClearEventVmcs(pVCpu);
8032	ASMSetFlags(pVmxTransient->uEflags);
8033	VMMRZCallRing3Enable(pVCpu);
8034	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8035	return VINF_EM_RAW_TO_R3;
8036	}
8037	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8038	{
8039	hmR0VmxClearEventVmcs(pVCpu);
8040	ASMSetFlags(pVmxTransient->uEflags);
8041	VMMRZCallRing3Enable(pVCpu);
8042	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8043	return VINF_EM_RAW_INTERRUPT;
8044	}
8045
8046	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8047	pVCpu->hm.s.Event.fPending = false;
8048
8049	return VINF_SUCCESS;
8050	}
8051
8052
8053	/**
8054	* Prepares to run guest code in VT-x and we've committed to doing so. This
8055	* means there is no backing out to ring-3 or anywhere else at this
8056	* point.
8057	*
8058	* @param pVM Pointer to the VM.
8059	* @param pVCpu Pointer to the VMCPU.
8060	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8061	* out-of-sync. Make sure to update the required fields
8062	* before using them.
8063	* @param pVmxTransient Pointer to the VMX transient structure.
8064	*
8065	* @remarks Called with preemption disabled.
8066	* @remarks No-long-jump zone!!!
8067	*/
8068	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8069	{
8070	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8071	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8072	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8073
8074	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8075	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8076
8077	/*
8078	* If we are injecting events to a real-on-v86 mode guest, we may have to update
8079	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8080	* Reload only the necessary state, the assertion will catch if other parts of the code
8081	* change.
8082	*/
8083	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8084	{
8085	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8086	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8087	}
8088
8089	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8090	if (!CPUMIsGuestFPUStateActive(pVCpu))
8091	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8092	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8093	#endif
8094
8095	if ( pVCpu->hm.s.fUseGuestFpu
8096	&& !CPUMIsGuestFPUStateActive(pVCpu))
8097	{
8098	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8099	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8100	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8101	}
8102
8103	/*
8104	* The host MSR values the very first time around won't be updated, so we need to
8105	* fill those values in. Subsequently, it's updated as part of the host state.
8106	*/
8107	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8108	&& pVCpu->hm.s.vmx.cMsrs > 0)
8109	{
8110	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8111	}
8112
8113	/*
8114	* Load the host state bits as we may've been preempted (only happens when
8115	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8116	*/
8117	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8118	{
8119	/* This ASSUMES that pfnStartVM has been set up already. */
8120	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8121	AssertRC(rc);
8122	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8123	}
8124	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8125
8126	/*
8127	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8128	*/
8129	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8130	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8131	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8132
8133	/* Store status of the shared guest-host state at the time of VM-entry. */
8134	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8135	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8136	{
8137	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8138	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8139	}
8140	else
8141	#endif
8142	{
8143	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8144	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8145	}
8146	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8147
8148	/*
8149	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8150	*/
8151	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8152	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8153
8154	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8155	RTCPUID idCurrentCpu = pCpu->idCpu;
8156	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8157	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8158	{
8159	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8160	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8161	}
8162
8163	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8164	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8165	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8166	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8167
8168	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8169
8170	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8171	to start executing. */
8172
8173	/*
8174	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8175	*/
8176	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8177	{
8178	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8179	{
8180	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8181	AssertRC(rc2);
8182	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8183	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */);
8184	}
8185	else
8186	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8187	}
8188	#ifdef VBOX_STRICT
8189	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8190	#endif
8191	}
8192
8193
8194	/**
8195	* Performs some essential restoration of state after running guest code in
8196	* VT-x.
8197	*
8198	* @param pVM Pointer to the VM.
8199	* @param pVCpu Pointer to the VMCPU.
8200	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8201	* out-of-sync. Make sure to update the required fields
8202	* before using them.
8203	* @param pVmxTransient Pointer to the VMX transient structure.
8204	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8205	*
8206	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8207	*
8208	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8209	* unconditionally when it is safe to do so.
8210	*/
8211	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8212	{
8213	NOREF(pVM);
8214
8215	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8216
8217	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8218	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8219	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8220	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8221	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8222
8223	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8224	{
8225	/** @todo Find a way to fix hardcoding a guestimate. */
8226	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8227	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8228	}
8229
8230	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8231	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8232	Assert(!(ASMGetFlags() & X86_EFL_IF));
8233	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8234
8235	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8236	if (CPUMIsGuestFPUStateActive(pVCpu))
8237	{
8238	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8239	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8240	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8241	}
8242	#endif
8243
8244	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8245	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8246	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8247	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8248
8249	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8250	uint32_t uExitReason;
8251	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8252	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8253	AssertRC(rc);
8254	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8255	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8256
8257	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8258	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8259	{
8260	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8261	pVmxTransient->fVMEntryFailed));
8262	return;
8263	}
8264
8265	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8266	{
8267	/* Update the guest interruptibility-state from the VMCS. */
8268	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8269	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8270	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8271	AssertRC(rc);
8272	#endif
8273	/*
8274	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8275	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8276	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8277	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8278	*/
8279	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8280	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8281	{
8282	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8283	AssertRC(rc);
8284	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8285	}
8286	}
8287	}
8288
8289
8290
8291	/**
8292	* Runs the guest code using VT-x the normal way.
8293	*
8294	* @returns VBox status code.
8295	* @param pVM Pointer to the VM.
8296	* @param pVCpu Pointer to the VMCPU.
8297	* @param pCtx Pointer to the guest-CPU context.
8298	*
8299	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8300	*/
8301	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8302	{
8303	VMXTRANSIENT VmxTransient;
8304	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8305	int rc = VERR_INTERNAL_ERROR_5;
8306	uint32_t cLoops = 0;
8307
8308	for (;; cLoops++)
8309	{
8310	Assert(!HMR0SuspendPending());
8311	HMVMX_ASSERT_CPU_SAFE();
8312
8313	/* Preparatory work for running guest code, this may force us to return
8314	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8315	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8316	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8317	if (rc != VINF_SUCCESS)
8318	break;
8319
8320	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8321	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8322	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8323
8324	/* Restore any residual host-state and save any bits shared between host
8325	and guest into the guest-CPU state. Re-enables interrupts! */
8326	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8327
8328	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8329	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8330	{
8331	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8332	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8333	return rc;
8334	}
8335
8336	/* Handle the VM-exit. */
8337	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8338	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8339	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8340	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8341	HMVMX_START_EXIT_DISPATCH_PROF();
8342	#ifdef HMVMX_USE_FUNCTION_TABLE
8343	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8344	#else
8345	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8346	#endif
8347	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8348	if (rc != VINF_SUCCESS)
8349	break;
8350	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8351	{
8352	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8353	rc = VINF_EM_RAW_INTERRUPT;
8354	break;
8355	}
8356	}
8357
8358	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8359	return rc;
8360	}
8361
8362
8363	/**
8364	* Single steps guest code using VT-x.
8365	*
8366	* @returns VBox status code.
8367	* @param pVM Pointer to the VM.
8368	* @param pVCpu Pointer to the VMCPU.
8369	* @param pCtx Pointer to the guest-CPU context.
8370	*
8371	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8372	*/
8373	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8374	{
8375	VMXTRANSIENT VmxTransient;
8376	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8377	int rc = VERR_INTERNAL_ERROR_5;
8378	uint32_t cLoops = 0;
8379	uint16_t uCsStart = pCtx->cs.Sel;
8380	uint64_t uRipStart = pCtx->rip;
8381
8382	for (;; cLoops++)
8383	{
8384	Assert(!HMR0SuspendPending());
8385	HMVMX_ASSERT_CPU_SAFE();
8386
8387	/* Preparatory work for running guest code, this may force us to return
8388	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8389	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8390	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8391	if (rc != VINF_SUCCESS)
8392	break;
8393
8394	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8395	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8396	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8397
8398	/* Restore any residual host-state and save any bits shared between host
8399	and guest into the guest-CPU state. Re-enables interrupts! */
8400	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8401
8402	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8403	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8404	{
8405	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8406	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8407	return rc;
8408	}
8409
8410	/* Handle the VM-exit. */
8411	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8412	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8413	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8414	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8415	HMVMX_START_EXIT_DISPATCH_PROF();
8416	#ifdef HMVMX_USE_FUNCTION_TABLE
8417	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8418	#else
8419	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8420	#endif
8421	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8422	if (rc != VINF_SUCCESS)
8423	break;
8424	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8425	{
8426	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8427	rc = VINF_EM_RAW_INTERRUPT;
8428	break;
8429	}
8430
8431	/*
8432	* Did the RIP change, if so, consider it a single step.
8433	* Otherwise, make sure one of the TFs gets set.
8434	*/
8435	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8436	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8437	AssertRCReturn(rc2, rc2);
8438	if ( pCtx->rip != uRipStart
8439	\|\| pCtx->cs.Sel != uCsStart)
8440	{
8441	rc = VINF_EM_DBG_STEPPED;
8442	break;
8443	}
8444	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8445	}
8446
8447	/*
8448	* Clear the X86_EFL_TF if necessary.
8449	*/
8450	if (pVCpu->hm.s.fClearTrapFlag)
8451	{
8452	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8453	AssertRCReturn(rc2, rc2);
8454	pVCpu->hm.s.fClearTrapFlag = false;
8455	pCtx->eflags.Bits.u1TF = 0;
8456	}
8457	/** @todo there seems to be issues with the resume flag when the monitor trap
8458	* flag is pending without being used. Seen early in bios init when
8459	* accessing APIC page in prot mode. */
8460
8461	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8462	return rc;
8463	}
8464
8465
8466	/**
8467	* Runs the guest code using VT-x.
8468	*
8469	* @returns VBox status code.
8470	* @param pVM Pointer to the VM.
8471	* @param pVCpu Pointer to the VMCPU.
8472	* @param pCtx Pointer to the guest-CPU context.
8473	*/
8474	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8475	{
8476	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8477	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8478	HMVMX_ASSERT_PREEMPT_SAFE();
8479
8480	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8481
8482	int rc;
8483	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8484	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8485	else
8486	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8487
8488	if (rc == VERR_EM_INTERPRETER)
8489	rc = VINF_EM_RAW_EMULATE_INSTR;
8490	else if (rc == VINF_EM_RESET)
8491	rc = VINF_EM_TRIPLE_FAULT;
8492
8493	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8494	if (RT_FAILURE(rc2))
8495	{
8496	pVCpu->hm.s.u32HMError = rc;
8497	rc = rc2;
8498	}
8499	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8500	return rc;
8501	}
8502
8503
8504	#ifndef HMVMX_USE_FUNCTION_TABLE
8505	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8506	{
8507	#ifdef DEBUG_ramshankar
8508	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8509	# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8510	#endif
8511	int rc;
8512	switch (rcReason)
8513	{
8514	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8515	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8516	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8517	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8518	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8519	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8520	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8521	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8522	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8523	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8524	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8525	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8526	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8527	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8528	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8529	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8530	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8531	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8532	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8533	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8534	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8535	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8536	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8537	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8538	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8539	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8540	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8541	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8542	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8543	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8544	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8545	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8546	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8547
8548	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8549	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8550	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8551	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8552	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8553	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8554	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8555	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8556	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8557
8558	case VMX_EXIT_VMCALL:
8559	case VMX_EXIT_VMCLEAR:
8560	case VMX_EXIT_VMLAUNCH:
8561	case VMX_EXIT_VMPTRLD:
8562	case VMX_EXIT_VMPTRST:
8563	case VMX_EXIT_VMREAD:
8564	case VMX_EXIT_VMRESUME:
8565	case VMX_EXIT_VMWRITE:
8566	case VMX_EXIT_VMXOFF:
8567	case VMX_EXIT_VMXON:
8568	case VMX_EXIT_INVEPT:
8569	case VMX_EXIT_INVVPID:
8570	case VMX_EXIT_VMFUNC:
8571	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8572	break;
8573	default:
8574	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8575	break;
8576	}
8577	return rc;
8578	}
8579	#endif
8580
8581	#ifdef DEBUG
8582	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8583	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8584	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8585
8586	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8587	do \
8588	{ \
8589	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8590	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8591	} while (0)
8592
8593	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8594	do { \
8595	AssertPtr(pVCpu); \
8596	AssertPtr(pMixedCtx); \
8597	AssertPtr(pVmxTransient); \
8598	Assert(pVmxTransient->fVMEntryFailed == false); \
8599	Assert(ASMIntAreEnabled()); \
8600	HMVMX_ASSERT_PREEMPT_SAFE(); \
8601	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8602	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8603	HMVMX_ASSERT_PREEMPT_SAFE(); \
8604	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8605	HMVMX_ASSERT_PREEMPT_CPUID(); \
8606	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8607	} while (0)
8608
8609	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8610	do { \
8611	Log4Func(("\n")); \
8612	} while (0)
8613	#else /* Release builds */
8614	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8615	do { \
8616	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8617	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8618	} while (0)
8619	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8620	#endif
8621
8622
8623	/**
8624	* Advances the guest RIP after reading it from the VMCS.
8625	*
8626	* @returns VBox status code.
8627	* @param pVCpu Pointer to the VMCPU.
8628	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8629	* out-of-sync. Make sure to update the required fields
8630	* before using them.
8631	* @param pVmxTransient Pointer to the VMX transient structure.
8632	*
8633	* @remarks No-long-jump zone!!!
8634	*/
8635	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8636	{
8637	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8638	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8639	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8640	AssertRCReturn(rc, rc);
8641
8642	pMixedCtx->rip += pVmxTransient->cbInstr;
8643	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8644
8645	/*
8646	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
8647	* pending debug exception field as it takes care of priority of events.
8648	*
8649	* See Intel spec. 32.2.1 "Debug Exceptions".
8650	*/
8651	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
8652
8653	return rc;
8654	}
8655
8656
8657	/**
8658	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8659	* and update error record fields accordingly.
8660	*
8661	* @return VMX_IGS_* return codes.
8662	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8663	* wrong with the guest state.
8664	*
8665	* @param pVM Pointer to the VM.
8666	* @param pVCpu Pointer to the VMCPU.
8667	* @param pCtx Pointer to the guest-CPU state.
8668	*/
8669	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8670	{
8671	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8672	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8673	uError = (err); \
8674	break; \
8675	} else do { } while (0)
8676
8677	int rc;
8678	uint32_t uError = VMX_IGS_ERROR;
8679	uint32_t u32Val;
8680	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8681
8682	do
8683	{
8684	/*
8685	* CR0.
8686	*/
8687	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8688	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8689	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8690	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8691	if (fUnrestrictedGuest)
8692	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8693
8694	uint32_t u32GuestCR0;
8695	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8696	AssertRCBreak(rc);
8697	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8698	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8699	if ( !fUnrestrictedGuest
8700	&& (u32GuestCR0 & X86_CR0_PG)
8701	&& !(u32GuestCR0 & X86_CR0_PE))
8702	{
8703	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8704	}
8705
8706	/*
8707	* CR4.
8708	*/
8709	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8710	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8711
8712	uint32_t u32GuestCR4;
8713	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8714	AssertRCBreak(rc);
8715	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8716	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8717
8718	/*
8719	* IA32_DEBUGCTL MSR.
8720	*/
8721	uint64_t u64Val;
8722	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8723	AssertRCBreak(rc);
8724	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8725	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8726	{
8727	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8728	}
8729	uint64_t u64DebugCtlMsr = u64Val;
8730
8731	#ifdef VBOX_STRICT
8732	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8733	AssertRCBreak(rc);
8734	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8735	#endif
8736	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8737
8738	/*
8739	* RIP and RFLAGS.
8740	*/
8741	uint32_t u32Eflags;
8742	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8743	if (HMVMX_IS_64BIT_HOST_MODE())
8744	{
8745	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8746	AssertRCBreak(rc);
8747	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8748	if ( !fLongModeGuest
8749	\|\| !pCtx->cs.Attr.n.u1Long)
8750	{
8751	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8752	}
8753	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8754	* must be identical if the "IA32e mode guest" VM-entry control is 1
8755	* and CS.L is 1. No check applies if the CPU supports 64
8756	* linear-address bits. */
8757
8758	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8759	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8760	AssertRCBreak(rc);
8761	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8762	VMX_IGS_RFLAGS_RESERVED);
8763	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8764	u32Eflags = u64Val;
8765	}
8766	else
8767	#endif
8768	{
8769	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8770	AssertRCBreak(rc);
8771	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8772	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8773	}
8774
8775	if ( fLongModeGuest
8776	\|\| ( fUnrestrictedGuest
8777	&& !(u32GuestCR0 & X86_CR0_PE)))
8778	{
8779	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8780	}
8781
8782	uint32_t u32EntryInfo;
8783	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8784	AssertRCBreak(rc);
8785	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8786	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8787	{
8788	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8789	}
8790
8791	/*
8792	* 64-bit checks.
8793	*/
8794	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8795	if (HMVMX_IS_64BIT_HOST_MODE())
8796	{
8797	if ( fLongModeGuest
8798	&& !fUnrestrictedGuest)
8799	{
8800	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8801	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8802	}
8803
8804	if ( !fLongModeGuest
8805	&& (u32GuestCR4 & X86_CR4_PCIDE))
8806	{
8807	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8808	}
8809
8810	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8811	* 51:32 beyond the processor's physical-address width are 0. */
8812
8813	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8814	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8815	{
8816	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8817	}
8818
8819	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8820	AssertRCBreak(rc);
8821	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8822
8823	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8824	AssertRCBreak(rc);
8825	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8826	}
8827	#endif
8828
8829	/*
8830	* PERF_GLOBAL MSR.
8831	*/
8832	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8833	{
8834	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8835	AssertRCBreak(rc);
8836	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8837	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8838	}
8839
8840	/*
8841	* PAT MSR.
8842	*/
8843	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8844	{
8845	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8846	AssertRCBreak(rc);
8847	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8848	for (unsigned i = 0; i < 8; i++)
8849	{
8850	uint8_t u8Val = (u64Val & 0x7);
8851	if ( u8Val != 0 /* UC */
8852	\|\| u8Val != 1 /* WC */
8853	\|\| u8Val != 4 /* WT */
8854	\|\| u8Val != 5 /* WP */
8855	\|\| u8Val != 6 /* WB */
8856	\|\| u8Val != 7 /* UC- */)
8857	{
8858	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8859	}
8860	u64Val >>= 3;
8861	}
8862	}
8863
8864	/*
8865	* EFER MSR.
8866	*/
8867	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8868	{
8869	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8870	AssertRCBreak(rc);
8871	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8872	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8873	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8874	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8875	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8876	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8877	}
8878
8879	/*
8880	* Segment registers.
8881	*/
8882	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8883	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8884	if (!(u32Eflags & X86_EFL_VM))
8885	{
8886	/* CS */
8887	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8888	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8889	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8890	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8891	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8892	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8893	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8894	/* CS cannot be loaded with NULL in protected mode. */
8895	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8896	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8897	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8898	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8899	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8900	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8901	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8902	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8903	else
8904	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8905
8906	/* SS */
8907	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8908	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8909	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8910	if ( !(pCtx->cr0 & X86_CR0_PE)
8911	\|\| pCtx->cs.Attr.n.u4Type == 3)
8912	{
8913	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8914	}
8915	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8916	{
8917	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8918	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8919	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8920	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8921	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8922	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8923	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8924	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8925	}
8926
8927	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8928	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8929	{
8930	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8931	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8932	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8933	\|\| pCtx->ds.Attr.n.u4Type > 11
8934	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8935	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8936	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8937	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8938	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8939	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8940	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8941	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8942	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8943	}
8944	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8945	{
8946	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8947	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8948	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8949	\|\| pCtx->es.Attr.n.u4Type > 11
8950	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8951	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8952	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8953	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8954	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8955	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8956	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8957	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8958	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8959	}
8960	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8961	{
8962	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8963	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8964	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8965	\|\| pCtx->fs.Attr.n.u4Type > 11
8966	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8967	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8968	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8969	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8970	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8971	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8972	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8973	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8974	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8975	}
8976	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8977	{
8978	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8979	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8980	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8981	\|\| pCtx->gs.Attr.n.u4Type > 11
8982	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8983	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8984	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8985	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8986	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8987	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8988	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8989	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8990	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8991	}
8992	/* 64-bit capable CPUs. */
8993	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8994	if (HMVMX_IS_64BIT_HOST_MODE())
8995	{
8996	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8997	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8998	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8999	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9000	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9001	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9002	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9003	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9004	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9005	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9006	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9007	}
9008	#endif
9009	}
9010	else
9011	{
9012	/* V86 mode checks. */
9013	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9014	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9015	{
9016	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9017	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9018	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9019	}
9020	else
9021	{
9022	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9023	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9024	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9025	}
9026
9027	/* CS */
9028	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9029	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9030	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9031	/* SS */
9032	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9033	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9034	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9035	/* DS */
9036	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9037	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9038	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9039	/* ES */
9040	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9041	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9042	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9043	/* FS */
9044	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9045	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9046	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9047	/* GS */
9048	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9049	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9050	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9051	/* 64-bit capable CPUs. */
9052	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9053	if (HMVMX_IS_64BIT_HOST_MODE())
9054	{
9055	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9056	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9057	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9058	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9059	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9060	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9061	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9062	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9063	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9064	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9065	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9066	}
9067	#endif
9068	}
9069
9070	/*
9071	* TR.
9072	*/
9073	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9074	/* 64-bit capable CPUs. */
9075	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9076	if (HMVMX_IS_64BIT_HOST_MODE())
9077	{
9078	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9079	}
9080	#endif
9081	if (fLongModeGuest)
9082	{
9083	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9084	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9085	}
9086	else
9087	{
9088	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9089	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9090	VMX_IGS_TR_ATTR_TYPE_INVALID);
9091	}
9092	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9093	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9094	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9095	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9096	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9097	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9098	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9099	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9100
9101	/*
9102	* GDTR and IDTR.
9103	*/
9104	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9105	if (HMVMX_IS_64BIT_HOST_MODE())
9106	{
9107	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9108	AssertRCBreak(rc);
9109	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9110
9111	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9112	AssertRCBreak(rc);
9113	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9114	}
9115	#endif
9116
9117	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9118	AssertRCBreak(rc);
9119	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9120
9121	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9122	AssertRCBreak(rc);
9123	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9124
9125	/*
9126	* Guest Non-Register State.
9127	*/
9128	/* Activity State. */
9129	uint32_t u32ActivityState;
9130	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9131	AssertRCBreak(rc);
9132	HMVMX_CHECK_BREAK( !u32ActivityState
9133	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9134	VMX_IGS_ACTIVITY_STATE_INVALID);
9135	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9136	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9137	uint32_t u32IntrState;
9138	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9139	AssertRCBreak(rc);
9140	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9141	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9142	{
9143	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9144	}
9145
9146	/** @todo Activity state and injecting interrupts. Left as a todo since we
9147	* currently don't use activity states but ACTIVE. */
9148
9149	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9150	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9151
9152	/* Guest interruptibility-state. */
9153	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9154	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9155	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9156	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9157	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9158	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9159	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9160	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9161	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9162	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9163	{
9164	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9165	{
9166	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9167	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9168	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9169	}
9170	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9171	{
9172	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9173	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9174	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9175	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9176	}
9177	}
9178	/** @todo Assumes the processor is not in SMM. */
9179	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9180	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9181	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9182	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9183	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9184	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9185	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9186	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9187	{
9188	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9189	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9190	}
9191
9192	/* Pending debug exceptions. */
9193	if (HMVMX_IS_64BIT_HOST_MODE())
9194	{
9195	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9196	AssertRCBreak(rc);
9197	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9198	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9199	u32Val = u64Val; /* For pending debug exceptions checks below. */
9200	}
9201	else
9202	{
9203	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9204	AssertRCBreak(rc);
9205	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9206	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9207	}
9208
9209	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9210	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9211	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9212	{
9213	if ( (u32Eflags & X86_EFL_TF)
9214	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9215	{
9216	/* Bit 14 is PendingDebug.BS. */
9217	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9218	}
9219	if ( !(u32Eflags & X86_EFL_TF)
9220	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9221	{
9222	/* Bit 14 is PendingDebug.BS. */
9223	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9224	}
9225	}
9226
9227	/* VMCS link pointer. */
9228	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9229	AssertRCBreak(rc);
9230	if (u64Val != UINT64_C(0xffffffffffffffff))
9231	{
9232	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9233	/** @todo Bits beyond the processor's physical-address width MBZ. */
9234	/** @todo 32-bit located in memory referenced by value of this field (as a
9235	* physical address) must contain the processor's VMCS revision ID. */
9236	/** @todo SMM checks. */
9237	}
9238
9239	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9240	* not using Nested Paging? */
9241	if ( pVM->hm.s.fNestedPaging
9242	&& !fLongModeGuest
9243	&& CPUMIsGuestInPAEModeEx(pCtx))
9244	{
9245	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9246	AssertRCBreak(rc);
9247	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9248
9249	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9250	AssertRCBreak(rc);
9251	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9252
9253	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9254	AssertRCBreak(rc);
9255	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9256
9257	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9258	AssertRCBreak(rc);
9259	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9260	}
9261
9262	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9263	if (uError == VMX_IGS_ERROR)
9264	uError = VMX_IGS_REASON_NOT_FOUND;
9265	} while (0);
9266
9267	pVCpu->hm.s.u32HMError = uError;
9268	return uError;
9269
9270	#undef HMVMX_ERROR_BREAK
9271	#undef HMVMX_CHECK_BREAK
9272	}
9273
9274	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9275	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9276	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9277
9278	/** @name VM-exit handlers.
9279	* @{
9280	*/
9281
9282	/**
9283	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9284	*/
9285	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9286	{
9287	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9288	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9289	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9290	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9291	return VINF_SUCCESS;
9292	return VINF_EM_RAW_INTERRUPT;
9293	}
9294
9295
9296	/**
9297	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9298	*/
9299	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9300	{
9301	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9302	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9303
9304	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9305	AssertRCReturn(rc, rc);
9306
9307	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9308	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9309	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9310	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9311
9312	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9313	{
9314	/*
9315	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9316	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9317	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9318	*
9319	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9320	*/
9321	VMXDispatchHostNmi();
9322	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9323	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9324	return VINF_SUCCESS;
9325	}
9326
9327	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9328	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9329	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9330	{
9331	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9332	return VINF_SUCCESS;
9333	}
9334	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9335	{
9336	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9337	return rc;
9338	}
9339
9340	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9341	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9342	switch (uIntType)
9343	{
9344	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9345	Assert(uVector == X86_XCPT_DB);
9346	/* no break */
9347	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9348	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9349	/* no break */
9350	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9351	{
9352	switch (uVector)
9353	{
9354	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9355	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9356	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9357	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9358	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9359	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9360	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9361	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9362	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9363	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9364	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9365	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9366	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9367	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9368	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9369	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9370	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9371	#endif
9372	default:
9373	{
9374	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9375	AssertRCReturn(rc, rc);
9376
9377	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9378	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9379	{
9380	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9381	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9382	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9383
9384	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9385	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9386	AssertRCReturn(rc, rc);
9387	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9388	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9389	0 /* GCPtrFaultAddress */);
9390	AssertRCReturn(rc, rc);
9391	}
9392	else
9393	{
9394	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9395	pVCpu->hm.s.u32HMError = uVector;
9396	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9397	}
9398	break;
9399	}
9400	}
9401	break;
9402	}
9403
9404	default:
9405	{
9406	pVCpu->hm.s.u32HMError = uExitIntInfo;
9407	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9408	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9409	break;
9410	}
9411	}
9412	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9413	return rc;
9414	}
9415
9416
9417	/**
9418	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9419	*/
9420	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9421	{
9422	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9423
9424	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9425	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9426	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9427	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9428	AssertRCReturn(rc, rc);
9429
9430	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9431	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9432	return VINF_SUCCESS;
9433	}
9434
9435
9436	/**
9437	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9438	*/
9439	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9440	{
9441	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9442	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9443	HMVMX_RETURN_UNEXPECTED_EXIT();
9444	}
9445
9446
9447	/**
9448	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9449	*/
9450	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9451	{
9452	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9453	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9454	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9455	}
9456
9457
9458	/**
9459	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9460	*/
9461	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9462	{
9463	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9464	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9465	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9466	}
9467
9468
9469	/**
9470	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9471	*/
9472	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9473	{
9474	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9475	PVM pVM = pVCpu->CTX_SUFF(pVM);
9476	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9477	if (RT_LIKELY(rc == VINF_SUCCESS))
9478	{
9479	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9480	Assert(pVmxTransient->cbInstr == 2);
9481	}
9482	else
9483	{
9484	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9485	rc = VERR_EM_INTERPRETER;
9486	}
9487	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9488	return rc;
9489	}
9490
9491
9492	/**
9493	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9494	*/
9495	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9496	{
9497	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9498	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9499	AssertRCReturn(rc, rc);
9500
9501	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9502	return VINF_EM_RAW_EMULATE_INSTR;
9503
9504	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9505	HMVMX_RETURN_UNEXPECTED_EXIT();
9506	}
9507
9508
9509	/**
9510	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9511	*/
9512	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9513	{
9514	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9515	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9516	AssertRCReturn(rc, rc);
9517
9518	PVM pVM = pVCpu->CTX_SUFF(pVM);
9519	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9520	if (RT_LIKELY(rc == VINF_SUCCESS))
9521	{
9522	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9523	Assert(pVmxTransient->cbInstr == 2);
9524	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9525	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9526	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9527	}
9528	else
9529	{
9530	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9531	rc = VERR_EM_INTERPRETER;
9532	}
9533	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9534	return rc;
9535	}
9536
9537
9538	/**
9539	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9540	*/
9541	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9542	{
9543	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9544	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9545	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9546	AssertRCReturn(rc, rc);
9547
9548	PVM pVM = pVCpu->CTX_SUFF(pVM);
9549	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9550	if (RT_LIKELY(rc == VINF_SUCCESS))
9551	{
9552	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9553	Assert(pVmxTransient->cbInstr == 3);
9554	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9555	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9556	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9557	}
9558	else
9559	{
9560	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9561	rc = VERR_EM_INTERPRETER;
9562	}
9563	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9564	return rc;
9565	}
9566
9567
9568	/**
9569	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9570	*/
9571	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9572	{
9573	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9574	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9575	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9576	AssertRCReturn(rc, rc);
9577
9578	PVM pVM = pVCpu->CTX_SUFF(pVM);
9579	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9580	if (RT_LIKELY(rc == VINF_SUCCESS))
9581	{
9582	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9583	Assert(pVmxTransient->cbInstr == 2);
9584	}
9585	else
9586	{
9587	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9588	rc = VERR_EM_INTERPRETER;
9589	}
9590	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9591	return rc;
9592	}
9593
9594
9595	/**
9596	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9597	*/
9598	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9599	{
9600	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9601	PVM pVM = pVCpu->CTX_SUFF(pVM);
9602	Assert(!pVM->hm.s.fNestedPaging);
9603
9604	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9605	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9606	AssertRCReturn(rc, rc);
9607
9608	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9609	rc = VBOXSTRICTRC_VAL(rc2);
9610	if (RT_LIKELY(rc == VINF_SUCCESS))
9611	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9612	else
9613	{
9614	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9615	pVmxTransient->uExitQualification, rc));
9616	}
9617	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9618	return rc;
9619	}
9620
9621
9622	/**
9623	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9624	*/
9625	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9626	{
9627	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9628	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9629	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9630	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9631	AssertRCReturn(rc, rc);
9632
9633	PVM pVM = pVCpu->CTX_SUFF(pVM);
9634	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9635	if (RT_LIKELY(rc == VINF_SUCCESS))
9636	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9637	else
9638	{
9639	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9640	rc = VERR_EM_INTERPRETER;
9641	}
9642	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9643	return rc;
9644	}
9645
9646
9647	/**
9648	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9649	*/
9650	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9651	{
9652	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9653	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9654	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9655	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9656	AssertRCReturn(rc, rc);
9657
9658	PVM pVM = pVCpu->CTX_SUFF(pVM);
9659	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9660	rc = VBOXSTRICTRC_VAL(rc2);
9661	if (RT_LIKELY( rc == VINF_SUCCESS
9662	\|\| rc == VINF_EM_HALT))
9663	{
9664	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9665	AssertRCReturn(rc3, rc3);
9666
9667	if ( rc == VINF_EM_HALT
9668	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9669	{
9670	rc = VINF_SUCCESS;
9671	}
9672	}
9673	else
9674	{
9675	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9676	rc = VERR_EM_INTERPRETER;
9677	}
9678	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9679	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9680	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9681	return rc;
9682	}
9683
9684
9685	/**
9686	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9687	*/
9688	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9689	{
9690	/*
9691	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9692	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9693	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9694	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9695	*/
9696	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9697	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9698	HMVMX_RETURN_UNEXPECTED_EXIT();
9699	}
9700
9701
9702	/**
9703	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9704	*/
9705	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9706	{
9707	/*
9708	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9709	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9710	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9711	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9712	*/
9713	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9714	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9715	HMVMX_RETURN_UNEXPECTED_EXIT();
9716	}
9717
9718
9719	/**
9720	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9721	*/
9722	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9723	{
9724	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9725	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9726	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9727	HMVMX_RETURN_UNEXPECTED_EXIT();
9728	}
9729
9730
9731	/**
9732	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9733	*/
9734	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9735	{
9736	/*
9737	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9738	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9739	* See Intel spec. 25.3 "Other Causes of VM-exits".
9740	*/
9741	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9742	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9743	HMVMX_RETURN_UNEXPECTED_EXIT();
9744	}
9745
9746
9747	/**
9748	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9749	* VM-exit.
9750	*/
9751	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9752	{
9753	/*
9754	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9755	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9756	*
9757	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9758	* See Intel spec. "23.8 Restrictions on VMX operation".
9759	*/
9760	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9761	return VINF_SUCCESS;
9762	}
9763
9764
9765	/**
9766	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9767	* VM-exit.
9768	*/
9769	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9770	{
9771	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9772	return VINF_EM_RESET;
9773	}
9774
9775
9776	/**
9777	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9778	*/
9779	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9780	{
9781	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9782	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9783	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9784	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9785	AssertRCReturn(rc, rc);
9786
9787	pMixedCtx->rip++;
9788	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9789	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9790	rc = VINF_SUCCESS;
9791	else
9792	rc = VINF_EM_HALT;
9793
9794	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9795	return rc;
9796	}
9797
9798
9799	/**
9800	* VM-exit handler for instructions that result in a #UD exception delivered to
9801	* the guest.
9802	*/
9803	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9804	{
9805	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9806	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9807	return VINF_SUCCESS;
9808	}
9809
9810
9811	/**
9812	* VM-exit handler for expiry of the VMX preemption timer.
9813	*/
9814	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9815	{
9816	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9817
9818	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9819	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9820
9821	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9822	PVM pVM = pVCpu->CTX_SUFF(pVM);
9823	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9824	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9825	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9826	}
9827
9828
9829	/**
9830	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9831	*/
9832	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9833	{
9834	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9835
9836	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9837	/** @todo check if XSETBV is supported by the recompiler. */
9838	return VERR_EM_INTERPRETER;
9839	}
9840
9841
9842	/**
9843	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9844	*/
9845	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9846	{
9847	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9848
9849	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9850	/** @todo implement EMInterpretInvpcid() */
9851	return VERR_EM_INTERPRETER;
9852	}
9853
9854
9855	/**
9856	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9857	* Error VM-exit.
9858	*/
9859	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9860	{
9861	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9862	AssertRCReturn(rc, rc);
9863
9864	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9865	NOREF(uInvalidReason);
9866
9867	#ifdef VBOX_STRICT
9868	uint32_t uIntrState;
9869	HMVMXHCUINTREG uHCReg;
9870	uint64_t u64Val;
9871	uint32_t u32Val;
9872
9873	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9874	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9875	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9876	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9877	AssertRCReturn(rc, rc);
9878
9879	Log4(("uInvalidReason %u\n", uInvalidReason));
9880	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9881	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9882	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9883	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9884
9885	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9886	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9887	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9888	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9889	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9890	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9891	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9892	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9893	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9894	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9895	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9896	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9897	#else
9898	NOREF(pVmxTransient);
9899	#endif
9900
9901	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9902	return VERR_VMX_INVALID_GUEST_STATE;
9903	}
9904
9905
9906	/**
9907	* VM-exit handler for VM-entry failure due to an MSR-load
9908	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9909	*/
9910	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9911	{
9912	NOREF(pVmxTransient);
9913	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9914	HMVMX_RETURN_UNEXPECTED_EXIT();
9915	}
9916
9917
9918	/**
9919	* VM-exit handler for VM-entry failure due to a machine-check event
9920	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9921	*/
9922	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9923	{
9924	NOREF(pVmxTransient);
9925	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9926	HMVMX_RETURN_UNEXPECTED_EXIT();
9927	}
9928
9929
9930	/**
9931	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9932	* theory.
9933	*/
9934	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9935	{
9936	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9937	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9938	return VERR_VMX_UNDEFINED_EXIT_CODE;
9939	}
9940
9941
9942	/**
9943	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9944	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9945	* Conditional VM-exit.
9946	*/
9947	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9948	{
9949	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9950
9951	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9952	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9953	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9954	return VERR_EM_INTERPRETER;
9955	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9956	HMVMX_RETURN_UNEXPECTED_EXIT();
9957	}
9958
9959
9960	/**
9961	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9962	*/
9963	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9964	{
9965	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9966
9967	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9968	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9969	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9970	return VERR_EM_INTERPRETER;
9971	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9972	HMVMX_RETURN_UNEXPECTED_EXIT();
9973	}
9974
9975
9976	/**
9977	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9978	*/
9979	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9980	{
9981	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9982
9983	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9984	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9985	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9986	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9987	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9988	{
9989	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9990	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9991	}
9992	AssertRCReturn(rc, rc);
9993	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9994
9995	#ifdef VBOX_STRICT
9996	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
9997	{
9998	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9999	{
10000	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10001	HMVMX_RETURN_UNEXPECTED_EXIT();
10002	}
10003	# if HC_ARCH_BITS == 64
10004	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10005	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10006	{
10007	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10008	HMVMX_RETURN_UNEXPECTED_EXIT();
10009	}
10010	# endif
10011	}
10012	#endif
10013
10014	PVM pVM = pVCpu->CTX_SUFF(pVM);
10015	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10016	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
10017	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10018	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10019
10020	if (RT_LIKELY(rc == VINF_SUCCESS))
10021	{
10022	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10023	Assert(pVmxTransient->cbInstr == 2);
10024	}
10025	return rc;
10026	}
10027
10028
10029	/**
10030	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10031	*/
10032	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10033	{
10034	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10035	PVM pVM = pVCpu->CTX_SUFF(pVM);
10036	int rc = VINF_SUCCESS;
10037
10038	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10039	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10040	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10041	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10042	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10043	{
10044	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10045	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10046	}
10047	AssertRCReturn(rc, rc);
10048	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10049
10050	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10051	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10052	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10053
10054	if (RT_LIKELY(rc == VINF_SUCCESS))
10055	{
10056	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10057
10058	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
10059	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10060	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10061	{
10062	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10063	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10064	EMInterpretWrmsr() changes it. */
10065	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10066	}
10067	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10068	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10069
10070	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10071	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10072	{
10073	switch (pMixedCtx->ecx)
10074	{
10075	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10076	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10077	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10078	case MSR_K8_FS_BASE: /* no break */
10079	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10080	default:
10081	{
10082	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10083	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10084	#if HC_ARCH_BITS == 64
10085	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10086	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10087	#endif
10088	break;
10089	}
10090	}
10091	}
10092	#ifdef VBOX_STRICT
10093	else
10094	{
10095	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10096	switch (pMixedCtx->ecx)
10097	{
10098	case MSR_IA32_SYSENTER_CS:
10099	case MSR_IA32_SYSENTER_EIP:
10100	case MSR_IA32_SYSENTER_ESP:
10101	case MSR_K8_FS_BASE:
10102	case MSR_K8_GS_BASE:
10103	{
10104	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10105	HMVMX_RETURN_UNEXPECTED_EXIT();
10106	}
10107
10108	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10109	default:
10110	{
10111	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10112	{
10113	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10114	pMixedCtx->ecx));
10115	HMVMX_RETURN_UNEXPECTED_EXIT();
10116	}
10117
10118	#if HC_ARCH_BITS == 64
10119	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10120	{
10121	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10122	HMVMX_RETURN_UNEXPECTED_EXIT();
10123	}
10124	#endif
10125	break;
10126	}
10127	}
10128	}
10129	#endif /* VBOX_STRICT */
10130	}
10131	return rc;
10132	}
10133
10134
10135	/**
10136	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10137	*/
10138	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10139	{
10140	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10141
10142	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10143	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10144	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10145	return VERR_EM_INTERPRETER;
10146	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10147	HMVMX_RETURN_UNEXPECTED_EXIT();
10148	}
10149
10150
10151	/**
10152	* VM-exit handler for when the TPR value is lowered below the specified
10153	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10154	*/
10155	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10156	{
10157	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10158	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10159
10160	/*
10161	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10162	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10163	* resume guest execution.
10164	*/
10165	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10166	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10167	return VINF_SUCCESS;
10168	}
10169
10170
10171	/**
10172	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10173	* VM-exit.
10174	*
10175	* @retval VINF_SUCCESS when guest execution can continue.
10176	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10177	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10178	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10179	* recompiler.
10180	*/
10181	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10182	{
10183	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10184	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10185	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10186	AssertRCReturn(rc, rc);
10187
10188	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10189	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10190	PVM pVM = pVCpu->CTX_SUFF(pVM);
10191	switch (uAccessType)
10192	{
10193	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10194	{
10195	#if 0
10196	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10197	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10198	#else
10199	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10200	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10201	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10202	#endif
10203	AssertRCReturn(rc, rc);
10204
10205	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10206	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10207	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10208	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
10209
10210	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10211	{
10212	case 0: /* CR0 */
10213	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10214	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10215	break;
10216	case 2: /* CR2 */
10217	/* Nothing to do here, CR2 it's not part of the VMCS. */
10218	break;
10219	case 3: /* CR3 */
10220	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10221	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10222	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10223	break;
10224	case 4: /* CR4 */
10225	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10226	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10227	break;
10228	case 8: /* CR8 */
10229	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10230	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10231	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10232	break;
10233	default:
10234	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10235	break;
10236	}
10237
10238	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10239	break;
10240	}
10241
10242	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10243	{
10244	/* EMInterpretCRxRead() requires EFER MSR, CS. */
10245	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10246	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10247	AssertRCReturn(rc, rc);
10248	Assert( !pVM->hm.s.fNestedPaging
10249	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10250	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10251
10252	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10253	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10254	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10255
10256	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10257	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10258	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10259	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10260	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10261	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10262	break;
10263	}
10264
10265	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10266	{
10267	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10268	AssertRCReturn(rc, rc);
10269	rc = EMInterpretCLTS(pVM, pVCpu);
10270	AssertRCReturn(rc, rc);
10271	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10272	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10273	Log4(("CRX CLTS write rc=%d\n", rc));
10274	break;
10275	}
10276
10277	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10278	{
10279	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10280	AssertRCReturn(rc, rc);
10281	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10282	if (RT_LIKELY(rc == VINF_SUCCESS))
10283	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10284	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10285	Log4(("CRX LMSW write rc=%d\n", rc));
10286	break;
10287	}
10288
10289	default:
10290	{
10291	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10292	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10293	}
10294	}
10295
10296	/* Validate possible error codes. */
10297	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10298	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10299	if (RT_SUCCESS(rc))
10300	{
10301	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10302	AssertRCReturn(rc2, rc2);
10303	}
10304
10305	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10306	return rc;
10307	}
10308
10309
10310	/**
10311	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10312	* VM-exit.
10313	*/
10314	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10315	{
10316	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10317	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10318
10319	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10320	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10321	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10322	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10323	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10324	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10325	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10326	AssertRCReturn(rc2, rc2);
10327
10328	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10329	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10330	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10331	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10332	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10333	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10334	bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10335	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10336
10337	/* I/O operation lookup arrays. */
10338	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10339	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10340
10341	VBOXSTRICTRC rcStrict;
10342	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10343	const uint32_t cbInstr = pVmxTransient->cbInstr;
10344	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10345	PVM pVM = pVCpu->CTX_SUFF(pVM);
10346	if (fIOString)
10347	{
10348	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10349	/*
10350	* INS/OUTS - I/O String instruction.
10351	*
10352	* Use instruction-information if available, otherwise fall back on
10353	* interpreting the instruction.
10354	*/
10355	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10356	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10357	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10358	{
10359	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10360	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10361	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10362	AssertRCReturn(rc2, rc2);
10363	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10364	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10365	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10366	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10367	if (fIOWrite)
10368	{
10369	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10370	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10371	}
10372	else
10373	{
10374	/*
10375	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10376	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10377	* See Intel Instruction spec. for "INS".
10378	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10379	*/
10380	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10381	}
10382	}
10383	else
10384	{
10385	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10386	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10387	AssertRCReturn(rc2, rc2);
10388	rcStrict = IEMExecOne(pVCpu);
10389	}
10390	/** @todo IEM needs to be setting these flags somehow. */
10391	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10392	fUpdateRipAlready = true;
10393	#else
10394	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10395	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10396	if (RT_SUCCESS(rcStrict))
10397	{
10398	if (fIOWrite)
10399	{
10400	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10401	(DISCPUMODE)pDis->uAddrMode, cbValue);
10402	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10403	}
10404	else
10405	{
10406	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10407	(DISCPUMODE)pDis->uAddrMode, cbValue);
10408	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10409	}
10410	}
10411	else
10412	{
10413	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10414	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10415	}
10416	#endif
10417	}
10418	else
10419	{
10420	/*
10421	* IN/OUT - I/O instruction.
10422	*/
10423	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10424	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10425	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10426	if (fIOWrite)
10427	{
10428	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10429	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10430	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10431	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10432	}
10433	else
10434	{
10435	uint32_t u32Result = 0;
10436	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10437	if (IOM_SUCCESS(rcStrict))
10438	{
10439	/* Save result of I/O IN instr. in AL/AX/EAX. */
10440	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10441	}
10442	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10443	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10444	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10445	}
10446	}
10447
10448	if (IOM_SUCCESS(rcStrict))
10449	{
10450	if (!fUpdateRipAlready)
10451	{
10452	pMixedCtx->rip += cbInstr;
10453	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10454	}
10455
10456	/*
10457	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10458	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10459	*/
10460	if (fIOString)
10461	{
10462	/** @todo Single-step for INS/OUTS with REP prefix? */
10463	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10464	}
10465	else if (fStepping)
10466	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
10467
10468	/*
10469	* If any I/O breakpoints are armed, we need to check if one triggered
10470	* and take appropriate action.
10471	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10472	*/
10473	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10474	AssertRCReturn(rc2, rc2);
10475
10476	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10477	* execution engines about whether hyper BPs and such are pending. */
10478	uint32_t const uDr7 = pMixedCtx->dr[7];
10479	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10480	&& X86_DR7_ANY_RW_IO(uDr7)
10481	&& (pMixedCtx->cr4 & X86_CR4_DE))
10482	\|\| DBGFBpIsHwIoArmed(pVM)))
10483	{
10484	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10485
10486	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10487	VMMRZCallRing3Disable(pVCpu);
10488	HM_DISABLE_PREEMPT_IF_NEEDED();
10489
10490	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10491
10492	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10493	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10494	{
10495	/* Raise #DB. */
10496	if (fIsGuestDbgActive)
10497	ASMSetDR6(pMixedCtx->dr[6]);
10498	if (pMixedCtx->dr[7] != uDr7)
10499	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10500
10501	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10502	}
10503	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10504	else if ( rcStrict2 != VINF_SUCCESS
10505	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10506	rcStrict = rcStrict2;
10507
10508	HM_RESTORE_PREEMPT_IF_NEEDED();
10509	VMMRZCallRing3Enable(pVCpu);
10510	}
10511	}
10512
10513	#ifdef DEBUG
10514	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10515	Assert(!fIOWrite);
10516	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10517	Assert(fIOWrite);
10518	else
10519	{
10520	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10521	* statuses, that the VMM device and some others may return. See
10522	* IOM_SUCCESS() for guidance. */
10523	AssertMsg( RT_FAILURE(rcStrict)
10524	\|\| rcStrict == VINF_SUCCESS
10525	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10526	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10527	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10528	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10529	}
10530	#endif
10531
10532	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10533	return VBOXSTRICTRC_TODO(rcStrict);
10534	}
10535
10536
10537	/**
10538	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10539	* VM-exit.
10540	*/
10541	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10542	{
10543	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10544
10545	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10546	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10547	AssertRCReturn(rc, rc);
10548	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10549	{
10550	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10551	AssertRCReturn(rc, rc);
10552	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10553	{
10554	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10555
10556	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10557	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10558	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10559	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10560	{
10561	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10562	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10563
10564	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10565	Assert(!pVCpu->hm.s.Event.fPending);
10566	pVCpu->hm.s.Event.fPending = true;
10567	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10568	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10569	AssertRCReturn(rc, rc);
10570	if (fErrorCodeValid)
10571	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10572	else
10573	pVCpu->hm.s.Event.u32ErrCode = 0;
10574	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10575	&& uVector == X86_XCPT_PF)
10576	{
10577	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10578	}
10579
10580	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10581	}
10582	}
10583	}
10584
10585	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10586	* emulation. */
10587	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10588	return VERR_EM_INTERPRETER;
10589	}
10590
10591
10592	/**
10593	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10594	*/
10595	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10596	{
10597	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10598	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10599	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10600	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10601	AssertRCReturn(rc, rc);
10602	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10603	return VINF_EM_DBG_STEPPED;
10604	}
10605
10606
10607	/**
10608	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10609	*/
10610	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10611	{
10612	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10613
10614	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10615	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10616	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10617	return VINF_SUCCESS;
10618	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10619	return rc;
10620
10621	#if 0
10622	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10623	* just sync the whole thing. */
10624	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10625	#else
10626	/* Aggressive state sync. for now. */
10627	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10628	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10629	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10630	#endif
10631	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10632	AssertRCReturn(rc, rc);
10633
10634	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10635	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10636	switch (uAccessType)
10637	{
10638	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10639	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10640	{
10641	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10642	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10643	{
10644	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10645	}
10646
10647	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10648	GCPhys &= PAGE_BASE_GC_MASK;
10649	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10650	PVM pVM = pVCpu->CTX_SUFF(pVM);
10651	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10652	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10653
10654	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10655	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10656	CPUMCTX2CORE(pMixedCtx), GCPhys);
10657	rc = VBOXSTRICTRC_VAL(rc2);
10658	Log4(("ApicAccess rc=%d\n", rc));
10659	if ( rc == VINF_SUCCESS
10660	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10661	\|\| rc == VERR_PAGE_NOT_PRESENT)
10662	{
10663	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10664	\| HM_CHANGED_GUEST_RSP
10665	\| HM_CHANGED_GUEST_RFLAGS
10666	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10667	rc = VINF_SUCCESS;
10668	}
10669	break;
10670	}
10671
10672	default:
10673	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10674	rc = VINF_EM_RAW_EMULATE_INSTR;
10675	break;
10676	}
10677
10678	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10679	return rc;
10680	}
10681
10682
10683	/**
10684	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10685	* VM-exit.
10686	*/
10687	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10688	{
10689	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10690
10691	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10692	if (pVmxTransient->fWasGuestDebugStateActive)
10693	{
10694	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10695	HMVMX_RETURN_UNEXPECTED_EXIT();
10696	}
10697
10698	int rc = VERR_INTERNAL_ERROR_5;
10699	if ( !DBGFIsStepping(pVCpu)
10700	&& !pVCpu->hm.s.fSingleInstruction
10701	&& !pVmxTransient->fWasHyperDebugStateActive)
10702	{
10703	/* Don't intercept MOV DRx and #DB any more. */
10704	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10705	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10706	AssertRCReturn(rc, rc);
10707
10708	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10709	{
10710	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10711	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10712	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10713	AssertRCReturn(rc, rc);
10714	#endif
10715	}
10716
10717	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10718	VMMRZCallRing3Disable(pVCpu);
10719	HM_DISABLE_PREEMPT_IF_NEEDED();
10720
10721	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10722	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10723	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10724
10725	HM_RESTORE_PREEMPT_IF_NEEDED();
10726	VMMRZCallRing3Enable(pVCpu);
10727
10728	#ifdef VBOX_WITH_STATISTICS
10729	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10730	AssertRCReturn(rc, rc);
10731	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10732	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10733	else
10734	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10735	#endif
10736	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10737	return VINF_SUCCESS;
10738	}
10739
10740	/*
10741	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10742	* Update the segment registers and DR7 from the CPU.
10743	*/
10744	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10745	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10746	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10747	AssertRCReturn(rc, rc);
10748	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10749
10750	PVM pVM = pVCpu->CTX_SUFF(pVM);
10751	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10752	{
10753	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10754	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10755	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10756	if (RT_SUCCESS(rc))
10757	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10758	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10759	}
10760	else
10761	{
10762	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10763	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10764	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10765	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10766	}
10767
10768	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10769	if (RT_SUCCESS(rc))
10770	{
10771	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10772	AssertRCReturn(rc2, rc2);
10773	}
10774	return rc;
10775	}
10776
10777
10778	/**
10779	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10780	* Conditional VM-exit.
10781	*/
10782	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10783	{
10784	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10785	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10786
10787	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10788	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10789	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10790	return VINF_SUCCESS;
10791	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10792	return rc;
10793
10794	RTGCPHYS GCPhys = 0;
10795	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10796
10797	#if 0
10798	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10799	#else
10800	/* Aggressive state sync. for now. */
10801	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10802	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10803	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10804	#endif
10805	AssertRCReturn(rc, rc);
10806
10807	/*
10808	* If we succeed, resume guest execution.
10809	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10810	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10811	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10812	* weird case. See @bugref{6043}.
10813	*/
10814	PVM pVM = pVCpu->CTX_SUFF(pVM);
10815	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10816	rc = VBOXSTRICTRC_VAL(rc2);
10817	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10818	if ( rc == VINF_SUCCESS
10819	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10820	\|\| rc == VERR_PAGE_NOT_PRESENT)
10821	{
10822	/* Successfully handled MMIO operation. */
10823	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10824	\| HM_CHANGED_GUEST_RSP
10825	\| HM_CHANGED_GUEST_RFLAGS
10826	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10827	rc = VINF_SUCCESS;
10828	}
10829	return rc;
10830	}
10831
10832
10833	/**
10834	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10835	* VM-exit.
10836	*/
10837	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10838	{
10839	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10840	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10841
10842	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10843	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10844	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10845	return VINF_SUCCESS;
10846	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10847	return rc;
10848
10849	RTGCPHYS GCPhys = 0;
10850	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10851	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10852	#if 0
10853	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10854	#else
10855	/* Aggressive state sync. for now. */
10856	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10857	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10858	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10859	#endif
10860	AssertRCReturn(rc, rc);
10861
10862	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10863	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10864
10865	RTGCUINT uErrorCode = 0;
10866	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10867	uErrorCode \|= X86_TRAP_PF_ID;
10868	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10869	uErrorCode \|= X86_TRAP_PF_RW;
10870	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10871	uErrorCode \|= X86_TRAP_PF_P;
10872
10873	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10874
10875	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10876	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10877
10878	/* Handle the pagefault trap for the nested shadow table. */
10879	PVM pVM = pVCpu->CTX_SUFF(pVM);
10880	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10881	TRPMResetTrap(pVCpu);
10882
10883	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10884	if ( rc == VINF_SUCCESS
10885	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10886	\|\| rc == VERR_PAGE_NOT_PRESENT)
10887	{
10888	/* Successfully synced our nested page tables. */
10889	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10890	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10891	\| HM_CHANGED_GUEST_RSP
10892	\| HM_CHANGED_GUEST_RFLAGS);
10893	return VINF_SUCCESS;
10894	}
10895
10896	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10897	return rc;
10898	}
10899
10900	/** @} */
10901
10902	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10903	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10904	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10905
10906	/** @name VM-exit exception handlers.
10907	* @{
10908	*/
10909
10910	/**
10911	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10912	*/
10913	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10914	{
10915	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10916	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10917
10918	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10919	AssertRCReturn(rc, rc);
10920
10921	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10922	{
10923	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
10924	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
10925
10926	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
10927	* provides VM-exit instruction length. If this causes problem later,
10928	* disassemble the instruction like it's done on AMD-V. */
10929	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10930	AssertRCReturn(rc2, rc2);
10931	return rc;
10932	}
10933
10934	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10935	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10936	return rc;
10937	}
10938
10939
10940	/**
10941	* VM-exit exception handler for #BP (Breakpoint exception).
10942	*/
10943	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10944	{
10945	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10946	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10947
10948	/** @todo Try optimize this by not saving the entire guest state unless
10949	* really needed. */
10950	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10951	AssertRCReturn(rc, rc);
10952
10953	PVM pVM = pVCpu->CTX_SUFF(pVM);
10954	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10955	if (rc == VINF_EM_RAW_GUEST_TRAP)
10956	{
10957	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10958	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10959	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10960	AssertRCReturn(rc, rc);
10961
10962	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10963	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10964	}
10965
10966	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10967	return rc;
10968	}
10969
10970
10971	/**
10972	* VM-exit exception handler for #DB (Debug exception).
10973	*/
10974	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10975	{
10976	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10977	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10978	Log6(("XcptDB\n"));
10979
10980	/*
10981	* Get the DR6-like values from the exit qualification and pass it to DBGF
10982	* for processing.
10983	*/
10984	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10985	AssertRCReturn(rc, rc);
10986
10987	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10988	uint64_t uDR6 = X86_DR6_INIT_VAL;
10989	uDR6 \|= ( pVmxTransient->uExitQualification
10990	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10991
10992	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10993	if (rc == VINF_EM_RAW_GUEST_TRAP)
10994	{
10995	/*
10996	* The exception was for the guest. Update DR6, DR7.GD and
10997	* IA32_DEBUGCTL.LBR before forwarding it.
10998	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10999	*/
11000	VMMRZCallRing3Disable(pVCpu);
11001	HM_DISABLE_PREEMPT_IF_NEEDED();
11002
11003	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11004	pMixedCtx->dr[6] \|= uDR6;
11005	if (CPUMIsGuestDebugStateActive(pVCpu))
11006	ASMSetDR6(pMixedCtx->dr[6]);
11007
11008	HM_RESTORE_PREEMPT_IF_NEEDED();
11009	VMMRZCallRing3Enable(pVCpu);
11010
11011	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11012	AssertRCReturn(rc, rc);
11013
11014	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11015	pMixedCtx->dr[7] &= ~X86_DR7_GD;
11016
11017	/* Paranoia. */
11018	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11019	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
11020
11021	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11022	AssertRCReturn(rc, rc);
11023
11024	/*
11025	* Raise #DB in the guest.
11026	*
11027	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11028	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11029	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11030	*
11031	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11032	*/
11033	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11034	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11035	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11036	AssertRCReturn(rc, rc);
11037	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11038	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11039	return VINF_SUCCESS;
11040	}
11041
11042	/*
11043	* Not a guest trap, must be a hypervisor related debug event then.
11044	* Update DR6 in case someone is interested in it.
11045	*/
11046	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11047	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11048	CPUMSetHyperDR6(pVCpu, uDR6);
11049
11050	return rc;
11051	}
11052
11053
11054	/**
11055	* VM-exit exception handler for #NM (Device-not-available exception: floating
11056	* point exception).
11057	*/
11058	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11059	{
11060	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11061
11062	/* We require CR0 and EFER. EFER is always up-to-date. */
11063	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11064	AssertRCReturn(rc, rc);
11065
11066	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11067	VMMRZCallRing3Disable(pVCpu);
11068	HM_DISABLE_PREEMPT_IF_NEEDED();
11069
11070	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11071	if (pVmxTransient->fWasGuestFPUStateActive)
11072	{
11073	rc = VINF_EM_RAW_GUEST_TRAP;
11074	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11075	}
11076	else
11077	{
11078	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11079	Assert(!pVmxTransient->fWasGuestFPUStateActive);
11080	#endif
11081	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11082	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11083	}
11084
11085	HM_RESTORE_PREEMPT_IF_NEEDED();
11086	VMMRZCallRing3Enable(pVCpu);
11087
11088	if (rc == VINF_SUCCESS)
11089	{
11090	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11091	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11092	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11093	pVCpu->hm.s.fUseGuestFpu = true;
11094	}
11095	else
11096	{
11097	/* Forward #NM to the guest. */
11098	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11099	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11100	AssertRCReturn(rc, rc);
11101	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11102	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11103	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11104	}
11105
11106	return VINF_SUCCESS;
11107	}
11108
11109
11110	/**
11111	* VM-exit exception handler for #GP (General-protection exception).
11112	*
11113	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11114	*/
11115	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11116	{
11117	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11118	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11119
11120	int rc = VERR_INTERNAL_ERROR_5;
11121	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11122	{
11123	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11124	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11125	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11126	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11127	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11128	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11129	AssertRCReturn(rc, rc);
11130	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11131	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11132	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11133	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11134	return rc;
11135	#else
11136	/* We don't intercept #GP. */
11137	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11138	NOREF(pVmxTransient);
11139	return VERR_VMX_UNEXPECTED_EXCEPTION;
11140	#endif
11141	}
11142
11143	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11144	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11145
11146	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11147	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11148	AssertRCReturn(rc, rc);
11149
11150	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11151	uint32_t cbOp = 0;
11152	PVM pVM = pVCpu->CTX_SUFF(pVM);
11153	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11154	if (RT_SUCCESS(rc))
11155	{
11156	rc = VINF_SUCCESS;
11157	Assert(cbOp == pDis->cbInstr);
11158	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11159	switch (pDis->pCurInstr->uOpcode)
11160	{
11161	case OP_CLI:
11162	{
11163	pMixedCtx->eflags.Bits.u1IF = 0;
11164	pMixedCtx->rip += pDis->cbInstr;
11165	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11166	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11167	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11168	break;
11169	}
11170
11171	case OP_STI:
11172	{
11173	pMixedCtx->eflags.Bits.u1IF = 1;
11174	pMixedCtx->rip += pDis->cbInstr;
11175	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11176	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11177	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11178	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11179	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11180	break;
11181	}
11182
11183	case OP_HLT:
11184	{
11185	rc = VINF_EM_HALT;
11186	pMixedCtx->rip += pDis->cbInstr;
11187	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11188	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11189	break;
11190	}
11191
11192	case OP_POPF:
11193	{
11194	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11195	uint32_t cbParm;
11196	uint32_t uMask;
11197	bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11198	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11199	{
11200	cbParm = 4;
11201	uMask = 0xffffffff;
11202	}
11203	else
11204	{
11205	cbParm = 2;
11206	uMask = 0xffff;
11207	}
11208
11209	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11210	RTGCPTR GCPtrStack = 0;
11211	X86EFLAGS Eflags;
11212	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11213	&GCPtrStack);
11214	if (RT_SUCCESS(rc))
11215	{
11216	Assert(sizeof(Eflags.u32) >= cbParm);
11217	Eflags.u32 = 0;
11218	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11219	}
11220	if (RT_FAILURE(rc))
11221	{
11222	rc = VERR_EM_INTERPRETER;
11223	break;
11224	}
11225	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11226	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11227	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11228	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11229	pMixedCtx->esp += cbParm;
11230	pMixedCtx->esp &= uMask;
11231	pMixedCtx->rip += pDis->cbInstr;
11232	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11233	\| HM_CHANGED_GUEST_RSP
11234	\| HM_CHANGED_GUEST_RFLAGS);
11235	/* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11236	if (fStepping)
11237	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11238
11239	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11240	break;
11241	}
11242
11243	case OP_PUSHF:
11244	{
11245	uint32_t cbParm;
11246	uint32_t uMask;
11247	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11248	{
11249	cbParm = 4;
11250	uMask = 0xffffffff;
11251	}
11252	else
11253	{
11254	cbParm = 2;
11255	uMask = 0xffff;
11256	}
11257
11258	/* Get the stack pointer & push the contents of eflags onto the stack. */
11259	RTGCPTR GCPtrStack = 0;
11260	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11261	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11262	if (RT_FAILURE(rc))
11263	{
11264	rc = VERR_EM_INTERPRETER;
11265	break;
11266	}
11267	X86EFLAGS Eflags = pMixedCtx->eflags;
11268	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11269	Eflags.Bits.u1RF = 0;
11270	Eflags.Bits.u1VM = 0;
11271
11272	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11273	if (RT_FAILURE(rc))
11274	{
11275	rc = VERR_EM_INTERPRETER;
11276	break;
11277	}
11278	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11279	pMixedCtx->esp -= cbParm;
11280	pMixedCtx->esp &= uMask;
11281	pMixedCtx->rip += pDis->cbInstr;
11282	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
11283	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11284	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11285	break;
11286	}
11287
11288	case OP_IRET:
11289	{
11290	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11291	* instruction reference. */
11292	RTGCPTR GCPtrStack = 0;
11293	uint32_t uMask = 0xffff;
11294	bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11295	uint16_t aIretFrame[3];
11296	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11297	{
11298	rc = VERR_EM_INTERPRETER;
11299	break;
11300	}
11301	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11302	&GCPtrStack);
11303	if (RT_SUCCESS(rc))
11304	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11305	if (RT_FAILURE(rc))
11306	{
11307	rc = VERR_EM_INTERPRETER;
11308	break;
11309	}
11310	pMixedCtx->eip = 0;
11311	pMixedCtx->ip = aIretFrame[0];
11312	pMixedCtx->cs.Sel = aIretFrame[1];
11313	pMixedCtx->cs.ValidSel = aIretFrame[1];
11314	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11315	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11316	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11317	pMixedCtx->sp += sizeof(aIretFrame);
11318	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11319	\| HM_CHANGED_GUEST_SEGMENT_REGS
11320	\| HM_CHANGED_GUEST_RSP
11321	\| HM_CHANGED_GUEST_RFLAGS);
11322	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11323	if (fStepping)
11324	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11325	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11326	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11327	break;
11328	}
11329
11330	case OP_INT:
11331	{
11332	uint16_t uVector = pDis->Param1.uValue & 0xff;
11333	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11334	/* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11335	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11336	break;
11337	}
11338
11339	case OP_INTO:
11340	{
11341	if (pMixedCtx->eflags.Bits.u1OF)
11342	{
11343	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11344	/* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11345	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11346	}
11347	break;
11348	}
11349
11350	default:
11351	{
11352	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11353	EMCODETYPE_SUPERVISOR);
11354	rc = VBOXSTRICTRC_VAL(rc2);
11355	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11356	/** @todo We have to set pending-debug exceptions here when the guest is
11357	* single-stepping depending on the instruction that was interpreted. */
11358	Log4(("#GP rc=%Rrc\n", rc));
11359	break;
11360	}
11361	}
11362	}
11363	else
11364	rc = VERR_EM_INTERPRETER;
11365
11366	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11367	("#GP Unexpected rc=%Rrc\n", rc));
11368	return rc;
11369	}
11370
11371
11372	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11373	/**
11374	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11375	* the exception reported in the VMX transient structure back into the VM.
11376	*
11377	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11378	* up-to-date.
11379	*/
11380	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11381	{
11382	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11383
11384	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11385	hmR0VmxCheckExitDueToEventDelivery(). */
11386	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11387	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11388	AssertRCReturn(rc, rc);
11389	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11390
11391	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11392	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11393	return VINF_SUCCESS;
11394	}
11395	#endif
11396
11397
11398	/**
11399	* VM-exit exception handler for #PF (Page-fault exception).
11400	*/
11401	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11402	{
11403	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11404	PVM pVM = pVCpu->CTX_SUFF(pVM);
11405	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11406	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11407	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11408	AssertRCReturn(rc, rc);
11409
11410	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11411	if (pVM->hm.s.fNestedPaging)
11412	{
11413	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11414	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11415	{
11416	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11417	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11418	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11419	}
11420	else
11421	{
11422	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11423	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11424	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11425	}
11426	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11427	return rc;
11428	}
11429	#else
11430	Assert(!pVM->hm.s.fNestedPaging);
11431	NOREF(pVM);
11432	#endif
11433
11434	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11435	AssertRCReturn(rc, rc);
11436
11437	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11438	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11439
11440	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11441	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11442	(RTGCPTR)pVmxTransient->uExitQualification);
11443
11444	Log4(("#PF: rc=%Rrc\n", rc));
11445	if (rc == VINF_SUCCESS)
11446	{
11447	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11448	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11449	* memory? We don't update the whole state here... */
11450	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11451	\| HM_CHANGED_GUEST_RSP
11452	\| HM_CHANGED_GUEST_RFLAGS
11453	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11454	TRPMResetTrap(pVCpu);
11455	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11456	return rc;
11457	}
11458	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11459	{
11460	if (!pVmxTransient->fVectoringPF)
11461	{
11462	/* It's a guest page fault and needs to be reflected to the guest. */
11463	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11464	TRPMResetTrap(pVCpu);
11465	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11466	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11467	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11468	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11469	}
11470	else
11471	{
11472	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11473	TRPMResetTrap(pVCpu);
11474	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11475	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11476	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11477	}
11478
11479	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11480	return VINF_SUCCESS;
11481	}
11482
11483	TRPMResetTrap(pVCpu);
11484	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11485	return rc;
11486	}
11487
11488	/** @} */
11489

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 50426

Download in other formats: