HMVMXR0.cpp@ 55810

Last change on this file since 55810 was 55757, checked in by vboxsync, 10 years ago
VMM/HMVMXR0: oops! not meant to commit this!
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 512.5 KB

Line
1	/* $Id: HMVMXR0.cpp 55757 2015-05-08 14:29:27Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2015 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/x86.h>
23	#include <iprt/asm-amd64-x86.h>
24	#include <iprt/thread.h>
25
26	#include <VBox/vmm/pdmapi.h>
27	#include <VBox/vmm/dbgf.h>
28	#include <VBox/vmm/iem.h>
29	#include <VBox/vmm/iom.h>
30	#include <VBox/vmm/selm.h>
31	#include <VBox/vmm/tm.h>
32	#include <VBox/vmm/gim.h>
33	#ifdef VBOX_WITH_REM
34	# include <VBox/vmm/rem.h>
35	#endif
36	#include "HMInternal.h"
37	#include <VBox/vmm/vm.h>
38	#include "HMVMXR0.h"
39	#include "dtrace/VBoxVMM.h"
40
41	#ifdef DEBUG_ramshankar
42	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
43	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
44	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
45	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
46	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
47	# define HMVMX_ALWAYS_TRAP_PF
48	# define HMVMX_ALWAYS_SWAP_FPU_STATE
49	# define HMVMX_ALWAYS_FLUSH_TLB
50	# define HMVMX_ALWAYS_SWAP_EFER
51	#endif
52
53
54	/*******************************************************************************
55	* Defined Constants And Macros *
56	*******************************************************************************/
57	#if defined(RT_ARCH_AMD64)
58	# define HMVMX_IS_64BIT_HOST_MODE() (true)
59	typedef RTHCUINTREG HMVMXHCUINTREG;
60	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
61	extern "C" uint32_t g_fVMXIs64bitHost;
62	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
63	typedef uint64_t HMVMXHCUINTREG;
64	#else
65	# define HMVMX_IS_64BIT_HOST_MODE() (false)
66	typedef RTHCUINTREG HMVMXHCUINTREG;
67	#endif
68
69	/** Use the function table. */
70	#define HMVMX_USE_FUNCTION_TABLE
71
72	/** Determine which tagged-TLB flush handler to use. */
73	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
74	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
75	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
76	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
77
78	/** @name Updated-guest-state flags.
79	* @{ */
80	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
81	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
82	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
83	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
84	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
85	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
86	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
87	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
88	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
89	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
90	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
91	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
92	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
93	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
94	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
95	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
96	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
97	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
98	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
99	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
100	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
101	\| HMVMX_UPDATED_GUEST_RSP \
102	\| HMVMX_UPDATED_GUEST_RFLAGS \
103	\| HMVMX_UPDATED_GUEST_CR0 \
104	\| HMVMX_UPDATED_GUEST_CR3 \
105	\| HMVMX_UPDATED_GUEST_CR4 \
106	\| HMVMX_UPDATED_GUEST_GDTR \
107	\| HMVMX_UPDATED_GUEST_IDTR \
108	\| HMVMX_UPDATED_GUEST_LDTR \
109	\| HMVMX_UPDATED_GUEST_TR \
110	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
111	\| HMVMX_UPDATED_GUEST_DEBUG \
112	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
113	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
114	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
115	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
116	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
117	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
118	\| HMVMX_UPDATED_GUEST_INTR_STATE \
119	\| HMVMX_UPDATED_GUEST_APIC_STATE)
120	/** @} */
121
122	/** @name
123	* Flags to skip redundant reads of some common VMCS fields that are not part of
124	* the guest-CPU state but are in the transient structure.
125	*/
126	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
127	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
128	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
129	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
130	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
131	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
132	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
133	/** @} */
134
135	/** @name
136	* States of the VMCS.
137	*
138	* This does not reflect all possible VMCS states but currently only those
139	* needed for maintaining the VMCS consistently even when thread-context hooks
140	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
141	*/
142	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
143	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
144	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
145	/** @} */
146
147	/**
148	* Exception bitmap mask for real-mode guests (real-on-v86).
149	*
150	* We need to intercept all exceptions manually except:
151	* - #NM, #MF handled in hmR0VmxLoadSharedCR0().
152	* - #DB handled in hmR0VmxLoadSharedDebugState().
153	* - #PF need not be intercepted even in real-mode if we have Nested Paging
154	* support.
155	*/
156	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
157	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
158	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
159	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
160	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
161	/* RT_BIT(X86_XCPT_MF) */ \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
162	\| RT_BIT(X86_XCPT_XF))
163
164	/**
165	* Exception bitmap mask for all contributory exceptions.
166	*
167	* Page fault is deliberately excluded here as it's conditional as to whether
168	* it's contributory or benign. Page faults are handled separately.
169	*/
170	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
171	\| RT_BIT(X86_XCPT_DE))
172
173	/** Maximum VM-instruction error number. */
174	#define HMVMX_INSTR_ERROR_MAX 28
175
176	/** Profiling macro. */
177	#ifdef HM_PROFILE_EXIT_DISPATCH
178	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
179	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
180	#else
181	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
182	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
183	#endif
184
185	/** Assert that preemption is disabled or covered by thread-context hooks. */
186	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
187	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
188
189	/** Assert that we haven't migrated CPUs when thread-context hooks are not
190	* used. */
191	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
192	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
193	("Illegal migration! Entered on CPU %u Current %u\n", \
194	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
195
196	/** Helper macro for VM-exit handlers called unexpectedly. */
197	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
198	do { \
199	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
200	return VERR_VMX_UNEXPECTED_EXIT; \
201	} while (0)
202
203
204	/*******************************************************************************
205	* Structures and Typedefs *
206	*******************************************************************************/
207	/**
208	* VMX transient state.
209	*
210	* A state structure for holding miscellaneous information across
211	* VMX non-root operation and restored after the transition.
212	*/
213	typedef struct VMXTRANSIENT
214	{
215	/** The host's rflags/eflags. */
216	RTCCUINTREG uEflags;
217	#if HC_ARCH_BITS == 32
218	uint32_t u32Alignment0;
219	#endif
220	/** The guest's TPR value used for TPR shadowing. */
221	uint8_t u8GuestTpr;
222	/** Alignment. */
223	uint8_t abAlignment0[7];
224
225	/** The basic VM-exit reason. */
226	uint16_t uExitReason;
227	/** Alignment. */
228	uint16_t u16Alignment0;
229	/** The VM-exit interruption error code. */
230	uint32_t uExitIntErrorCode;
231	/** The VM-exit exit code qualification. */
232	uint64_t uExitQualification;
233
234	/** The VM-exit interruption-information field. */
235	uint32_t uExitIntInfo;
236	/** The VM-exit instruction-length field. */
237	uint32_t cbInstr;
238	/** The VM-exit instruction-information field. */
239	union
240	{
241	/** Plain unsigned int representation. */
242	uint32_t u;
243	/** INS and OUTS information. */
244	struct
245	{
246	uint32_t u6Reserved0 : 7;
247	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
248	uint32_t u3AddrSize : 3;
249	uint32_t u5Reserved1 : 5;
250	/** The segment register (X86_SREG_XXX). */
251	uint32_t iSegReg : 3;
252	uint32_t uReserved2 : 14;
253	} StrIo;
254	} ExitInstrInfo;
255	/** Whether the VM-entry failed or not. */
256	bool fVMEntryFailed;
257	/** Alignment. */
258	uint8_t abAlignment1[3];
259
260	/** The VM-entry interruption-information field. */
261	uint32_t uEntryIntInfo;
262	/** The VM-entry exception error code field. */
263	uint32_t uEntryXcptErrorCode;
264	/** The VM-entry instruction length field. */
265	uint32_t cbEntryInstr;
266
267	/** IDT-vectoring information field. */
268	uint32_t uIdtVectoringInfo;
269	/** IDT-vectoring error code. */
270	uint32_t uIdtVectoringErrorCode;
271
272	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
273	uint32_t fVmcsFieldsRead;
274
275	/** Whether the guest FPU was active at the time of VM-exit. */
276	bool fWasGuestFPUStateActive;
277	/** Whether the guest debug state was active at the time of VM-exit. */
278	bool fWasGuestDebugStateActive;
279	/** Whether the hyper debug state was active at the time of VM-exit. */
280	bool fWasHyperDebugStateActive;
281	/** Whether TSC-offsetting should be setup before VM-entry. */
282	bool fUpdateTscOffsettingAndPreemptTimer;
283	/** Whether the VM-exit was caused by a page-fault during delivery of a
284	* contributory exception or a page-fault. */
285	bool fVectoringDoublePF;
286	/** Whether the VM-exit was caused by a page-fault during delivery of an
287	* external interrupt or NMI. */
288	bool fVectoringPF;
289	} VMXTRANSIENT;
290	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
291	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
292	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
293	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
294	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
295	/** Pointer to VMX transient state. */
296	typedef VMXTRANSIENT *PVMXTRANSIENT;
297
298
299	/**
300	* MSR-bitmap read permissions.
301	*/
302	typedef enum VMXMSREXITREAD
303	{
304	/** Reading this MSR causes a VM-exit. */
305	VMXMSREXIT_INTERCEPT_READ = 0xb,
306	/** Reading this MSR does not cause a VM-exit. */
307	VMXMSREXIT_PASSTHRU_READ
308	} VMXMSREXITREAD;
309	/** Pointer to MSR-bitmap read permissions. */
310	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
311
312	/**
313	* MSR-bitmap write permissions.
314	*/
315	typedef enum VMXMSREXITWRITE
316	{
317	/** Writing to this MSR causes a VM-exit. */
318	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
319	/** Writing to this MSR does not cause a VM-exit. */
320	VMXMSREXIT_PASSTHRU_WRITE
321	} VMXMSREXITWRITE;
322	/** Pointer to MSR-bitmap write permissions. */
323	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
324
325
326	/**
327	* VMX VM-exit handler.
328	*
329	* @returns VBox status code.
330	* @param pVCpu Pointer to the VMCPU.
331	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
332	* out-of-sync. Make sure to update the required
333	* fields before using them.
334	* @param pVmxTransient Pointer to the VMX-transient structure.
335	*/
336	#ifndef HMVMX_USE_FUNCTION_TABLE
337	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
338	#else
339	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
340	/** Pointer to VM-exit handler. */
341	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
342	#endif
343
344
345	/*******************************************************************************
346	* Internal Functions *
347	*******************************************************************************/
348	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
349	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
350	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
351	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
352	bool fStepping, uint32_t *puIntState);
353	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
354	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
355	#endif
356	#ifndef HMVMX_USE_FUNCTION_TABLE
357	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
358	# define HMVMX_EXIT_DECL static int
359	#else
360	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
361	#endif
362	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
363	uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
364
365	/** @name VM-exit handlers.
366	* @{
367	*/
368	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
369	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
370	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
371	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
372	static FNVMXEXITHANDLER hmR0VmxExitSipi;
373	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
374	static FNVMXEXITHANDLER hmR0VmxExitSmi;
375	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
376	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
377	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
378	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
379	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
380	static FNVMXEXITHANDLER hmR0VmxExitHlt;
381	static FNVMXEXITHANDLER hmR0VmxExitInvd;
382	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
383	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
384	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
385	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
386	static FNVMXEXITHANDLER hmR0VmxExitRsm;
387	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
388	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
389	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
390	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
391	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
392	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
393	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
394	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
395	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
396	static FNVMXEXITHANDLER hmR0VmxExitMwait;
397	static FNVMXEXITHANDLER hmR0VmxExitMtf;
398	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
399	static FNVMXEXITHANDLER hmR0VmxExitPause;
400	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
401	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
402	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
403	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
404	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
405	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
406	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
407	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
408	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
409	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
410	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
411	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
412	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
413	/** @} */
414
415	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
417	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
418	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
419	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
420	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
421	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
422	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
423	#endif
424	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
425
426	/*******************************************************************************
427	* Global Variables *
428	*******************************************************************************/
429	#ifdef HMVMX_USE_FUNCTION_TABLE
430
431	/**
432	* VMX_EXIT dispatch table.
433	*/
434	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
435	{
436	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
437	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
438	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
439	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
440	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
441	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
442	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
443	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
444	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
445	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
446	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
447	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
448	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
449	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
450	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
451	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
452	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
453	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
454	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
455	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
456	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
457	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
458	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
459	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
460	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
461	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
462	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
463	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
464	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
465	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
466	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
467	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
468	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
469	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
470	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
471	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
472	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
473	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
474	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
475	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
476	/* 40 UNDEFINED */ hmR0VmxExitPause,
477	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
478	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
479	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
480	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
481	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
482	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
483	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
484	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
485	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
486	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
487	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
488	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
489	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
490	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
491	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
492	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
493	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
494	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
495	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
496	/* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
497	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
498	/* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
499	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
500	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
501	};
502	#endif /* HMVMX_USE_FUNCTION_TABLE */
503
504	#ifdef VBOX_STRICT
505	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
506	{
507	/* 0 */ "(Not Used)",
508	/* 1 */ "VMCALL executed in VMX root operation.",
509	/* 2 */ "VMCLEAR with invalid physical address.",
510	/* 3 */ "VMCLEAR with VMXON pointer.",
511	/* 4 */ "VMLAUNCH with non-clear VMCS.",
512	/* 5 */ "VMRESUME with non-launched VMCS.",
513	/* 6 */ "VMRESUME after VMXOFF",
514	/* 7 */ "VM-entry with invalid control fields.",
515	/* 8 */ "VM-entry with invalid host state fields.",
516	/* 9 */ "VMPTRLD with invalid physical address.",
517	/* 10 */ "VMPTRLD with VMXON pointer.",
518	/* 11 */ "VMPTRLD with incorrect revision identifier.",
519	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
520	/* 13 */ "VMWRITE to read-only VMCS component.",
521	/* 14 */ "(Not Used)",
522	/* 15 */ "VMXON executed in VMX root operation.",
523	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
524	/* 17 */ "VM-entry with non-launched executing VMCS.",
525	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
526	/* 19 */ "VMCALL with non-clear VMCS.",
527	/* 20 */ "VMCALL with invalid VM-exit control fields.",
528	/* 21 */ "(Not Used)",
529	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
530	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
531	/* 24 */ "VMCALL with invalid SMM-monitor features.",
532	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
533	/* 26 */ "VM-entry with events blocked by MOV SS.",
534	/* 27 */ "(Not Used)",
535	/* 28 */ "Invalid operand to INVEPT/INVVPID."
536	};
537	#endif /* VBOX_STRICT */
538
539
540
541	/**
542	* Updates the VM's last error record. If there was a VMX instruction error,
543	* reads the error data from the VMCS and updates VCPU's last error record as
544	* well.
545	*
546	* @param pVM Pointer to the VM.
547	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
548	* VERR_VMX_UNABLE_TO_START_VM or
549	* VERR_VMX_INVALID_VMCS_FIELD).
550	* @param rc The error code.
551	*/
552	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
553	{
554	AssertPtr(pVM);
555	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
556	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
557	{
558	AssertPtrReturnVoid(pVCpu);
559	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
560	}
561	pVM->hm.s.lLastError = rc;
562	}
563
564
565	/**
566	* Reads the VM-entry interruption-information field from the VMCS into the VMX
567	* transient structure.
568	*
569	* @returns VBox status code.
570	* @param pVmxTransient Pointer to the VMX transient structure.
571	*
572	* @remarks No-long-jump zone!!!
573	*/
574	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
575	{
576	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
577	AssertRCReturn(rc, rc);
578	return VINF_SUCCESS;
579	}
580
581
582	/**
583	* Reads the VM-entry exception error code field from the VMCS into
584	* the VMX transient structure.
585	*
586	* @returns VBox status code.
587	* @param pVmxTransient Pointer to the VMX transient structure.
588	*
589	* @remarks No-long-jump zone!!!
590	*/
591	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
592	{
593	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
594	AssertRCReturn(rc, rc);
595	return VINF_SUCCESS;
596	}
597
598
599	/**
600	* Reads the VM-entry exception error code field from the VMCS into
601	* the VMX transient structure.
602	*
603	* @returns VBox status code.
604	* @param pVmxTransient Pointer to the VMX transient structure.
605	*
606	* @remarks No-long-jump zone!!!
607	*/
608	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
609	{
610	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
611	AssertRCReturn(rc, rc);
612	return VINF_SUCCESS;
613	}
614
615
616	/**
617	* Reads the VM-exit interruption-information field from the VMCS into the VMX
618	* transient structure.
619	*
620	* @returns VBox status code.
621	* @param pVmxTransient Pointer to the VMX transient structure.
622	*/
623	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
624	{
625	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
626	{
627	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
628	AssertRCReturn(rc, rc);
629	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
630	}
631	return VINF_SUCCESS;
632	}
633
634
635	/**
636	* Reads the VM-exit interruption error code from the VMCS into the VMX
637	* transient structure.
638	*
639	* @returns VBox status code.
640	* @param pVmxTransient Pointer to the VMX transient structure.
641	*/
642	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
643	{
644	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
645	{
646	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
647	AssertRCReturn(rc, rc);
648	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
649	}
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Reads the VM-exit instruction length field from the VMCS into the VMX
656	* transient structure.
657	*
658	* @returns VBox status code.
659	* @param pVCpu Pointer to the VMCPU.
660	* @param pVmxTransient Pointer to the VMX transient structure.
661	*/
662	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
663	{
664	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
665	{
666	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
667	AssertRCReturn(rc, rc);
668	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
669	}
670	return VINF_SUCCESS;
671	}
672
673
674	/**
675	* Reads the VM-exit instruction-information field from the VMCS into
676	* the VMX transient structure.
677	*
678	* @returns VBox status code.
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
684	{
685	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the exit code qualification from the VMCS into the VMX transient
695	* structure.
696	*
697	* @returns VBox status code.
698	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
699	* case).
700	* @param pVmxTransient Pointer to the VMX transient structure.
701	*/
702	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
703	{
704	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
705	{
706	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
707	AssertRCReturn(rc, rc);
708	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
709	}
710	return VINF_SUCCESS;
711	}
712
713
714	/**
715	* Reads the IDT-vectoring information field from the VMCS into the VMX
716	* transient structure.
717	*
718	* @returns VBox status code.
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*
721	* @remarks No-long-jump zone!!!
722	*/
723	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
724	{
725	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
726	{
727	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
728	AssertRCReturn(rc, rc);
729	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
730	}
731	return VINF_SUCCESS;
732	}
733
734
735	/**
736	* Reads the IDT-vectoring error code from the VMCS into the VMX
737	* transient structure.
738	*
739	* @returns VBox status code.
740	* @param pVmxTransient Pointer to the VMX transient structure.
741	*/
742	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
743	{
744	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
745	{
746	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
747	AssertRCReturn(rc, rc);
748	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
749	}
750	return VINF_SUCCESS;
751	}
752
753
754	/**
755	* Enters VMX root mode operation on the current CPU.
756	*
757	* @returns VBox status code.
758	* @param pVM Pointer to the VM (optional, can be NULL, after
759	* a resume).
760	* @param HCPhysCpuPage Physical address of the VMXON region.
761	* @param pvCpuPage Pointer to the VMXON region.
762	*/
763	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
764	{
765	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
766	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
767	Assert(pvCpuPage);
768	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
769
770	if (pVM)
771	{
772	/* Write the VMCS revision dword to the VMXON region. */
773	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
774	}
775
776	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
777	RTCCUINTREG uEflags = ASMIntDisableFlags();
778
779	/* Enable the VMX bit in CR4 if necessary. */
780	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
781
782	/* Enter VMX root mode. */
783	int rc = VMXEnable(HCPhysCpuPage);
784	if (RT_FAILURE(rc))
785	{
786	if (!(uOldCr4 & X86_CR4_VMXE))
787	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
788
789	if (pVM)
790	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
791	}
792
793	/* Restore interrupts. */
794	ASMSetFlags(uEflags);
795	return rc;
796	}
797
798
799	/**
800	* Exits VMX root mode operation on the current CPU.
801	*
802	* @returns VBox status code.
803	*/
804	static int hmR0VmxLeaveRootMode(void)
805	{
806	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
807
808	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
809	RTCCUINTREG uEflags = ASMIntDisableFlags();
810
811	/* If we're for some reason not in VMX root mode, then don't leave it. */
812	RTCCUINTREG uHostCR4 = ASMGetCR4();
813
814	int rc;
815	if (uHostCR4 & X86_CR4_VMXE)
816	{
817	/* Exit VMX root mode and clear the VMX bit in CR4. */
818	VMXDisable();
819	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
820	rc = VINF_SUCCESS;
821	}
822	else
823	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
824
825	/* Restore interrupts. */
826	ASMSetFlags(uEflags);
827	return rc;
828	}
829
830
831	/**
832	* Allocates and maps one physically contiguous page. The allocated page is
833	* zero'd out. (Used by various VT-x structures).
834	*
835	* @returns IPRT status code.
836	* @param pMemObj Pointer to the ring-0 memory object.
837	* @param ppVirt Where to store the virtual address of the
838	* allocation.
839	* @param pPhys Where to store the physical address of the
840	* allocation.
841	*/
842	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
843	{
844	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
845	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
846	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
847
848	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
849	if (RT_FAILURE(rc))
850	return rc;
851	ppVirt = RTR0MemObjAddress(pMemObj);
852	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
853	ASMMemZero32(*ppVirt, PAGE_SIZE);
854	return VINF_SUCCESS;
855	}
856
857
858	/**
859	* Frees and unmaps an allocated physical page.
860	*
861	* @param pMemObj Pointer to the ring-0 memory object.
862	* @param ppVirt Where to re-initialize the virtual address of
863	* allocation as 0.
864	* @param pHCPhys Where to re-initialize the physical address of the
865	* allocation as 0.
866	*/
867	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
868	{
869	AssertPtr(pMemObj);
870	AssertPtr(ppVirt);
871	AssertPtr(pHCPhys);
872	if (*pMemObj != NIL_RTR0MEMOBJ)
873	{
874	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
875	AssertRC(rc);
876	*pMemObj = NIL_RTR0MEMOBJ;
877	*ppVirt = 0;
878	*pHCPhys = 0;
879	}
880	}
881
882
883	/**
884	* Worker function to free VT-x related structures.
885	*
886	* @returns IPRT status code.
887	* @param pVM Pointer to the VM.
888	*/
889	static void hmR0VmxStructsFree(PVM pVM)
890	{
891	for (VMCPUID i = 0; i < pVM->cCpus; i++)
892	{
893	PVMCPU pVCpu = &pVM->aCpus[i];
894	AssertPtr(pVCpu);
895
896	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
897	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
898
899	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
900	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
901
902	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
903	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
904	}
905
906	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
907	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
908	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
909	#endif
910	}
911
912
913	/**
914	* Worker function to allocate VT-x related VM structures.
915	*
916	* @returns IPRT status code.
917	* @param pVM Pointer to the VM.
918	*/
919	static int hmR0VmxStructsAlloc(PVM pVM)
920	{
921	/*
922	* Initialize members up-front so we can cleanup properly on allocation failure.
923	*/
924	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
925	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
926	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
927	pVM->hm.s.vmx.HCPhys##a_Name = 0;
928
929	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
930	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
931	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
932	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
933
934	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
935	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
936	#endif
937	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
938
939	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
940	for (VMCPUID i = 0; i < pVM->cCpus; i++)
941	{
942	PVMCPU pVCpu = &pVM->aCpus[i];
943	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
944	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
945	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
946	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
947	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
948	}
949	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
950	#undef VMXLOCAL_INIT_VM_MEMOBJ
951
952	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
953	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
954	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
955	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
956
957	/*
958	* Allocate all the VT-x structures.
959	*/
960	int rc = VINF_SUCCESS;
961	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
962	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
963	if (RT_FAILURE(rc))
964	goto cleanup;
965	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
966	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
967	#endif
968
969	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
970	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
971	{
972	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
973	&pVM->hm.s.vmx.HCPhysApicAccess);
974	if (RT_FAILURE(rc))
975	goto cleanup;
976	}
977
978	/*
979	* Initialize per-VCPU VT-x structures.
980	*/
981	for (VMCPUID i = 0; i < pVM->cCpus; i++)
982	{
983	PVMCPU pVCpu = &pVM->aCpus[i];
984	AssertPtr(pVCpu);
985
986	/* Allocate the VM control structure (VMCS). */
987	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
988	if (RT_FAILURE(rc))
989	goto cleanup;
990
991	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
992	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
993	{
994	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
995	&pVCpu->hm.s.vmx.HCPhysVirtApic);
996	if (RT_FAILURE(rc))
997	goto cleanup;
998	}
999
1000	/*
1001	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1002	* transparent accesses of specific MSRs.
1003	*
1004	* If the condition for enabling MSR bitmaps changes here, don't forget to
1005	* update HMIsMsrBitmapsAvailable().
1006	*/
1007	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1008	{
1009	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1010	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1011	if (RT_FAILURE(rc))
1012	goto cleanup;
1013	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1014	}
1015
1016	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1017	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1018	if (RT_FAILURE(rc))
1019	goto cleanup;
1020
1021	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1022	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1023	if (RT_FAILURE(rc))
1024	goto cleanup;
1025	}
1026
1027	return VINF_SUCCESS;
1028
1029	cleanup:
1030	hmR0VmxStructsFree(pVM);
1031	return rc;
1032	}
1033
1034
1035	/**
1036	* Does global VT-x initialization (called during module initialization).
1037	*
1038	* @returns VBox status code.
1039	*/
1040	VMMR0DECL(int) VMXR0GlobalInit(void)
1041	{
1042	#ifdef HMVMX_USE_FUNCTION_TABLE
1043	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1044	# ifdef VBOX_STRICT
1045	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1046	Assert(g_apfnVMExitHandlers[i]);
1047	# endif
1048	#endif
1049	return VINF_SUCCESS;
1050	}
1051
1052
1053	/**
1054	* Does global VT-x termination (called during module termination).
1055	*/
1056	VMMR0DECL(void) VMXR0GlobalTerm()
1057	{
1058	/* Nothing to do currently. */
1059	}
1060
1061
1062	/**
1063	* Sets up and activates VT-x on the current CPU.
1064	*
1065	* @returns VBox status code.
1066	* @param pCpu Pointer to the global CPU info struct.
1067	* @param pVM Pointer to the VM (can be NULL after a host resume
1068	* operation).
1069	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1070	* fEnabledByHost is true).
1071	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1072	* @a fEnabledByHost is true).
1073	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1074	* enable VT-x on the host.
1075	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1076	*/
1077	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1078	void *pvMsrs)
1079	{
1080	Assert(pCpu);
1081	Assert(pvMsrs);
1082	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1083
1084	/* Enable VT-x if it's not already enabled by the host. */
1085	if (!fEnabledByHost)
1086	{
1087	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1088	if (RT_FAILURE(rc))
1089	return rc;
1090	}
1091
1092	/*
1093	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1094	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1095	*/
1096	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1097	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1098	{
1099	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1100	pCpu->fFlushAsidBeforeUse = false;
1101	}
1102	else
1103	pCpu->fFlushAsidBeforeUse = true;
1104
1105	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1106	++pCpu->cTlbFlushes;
1107
1108	return VINF_SUCCESS;
1109	}
1110
1111
1112	/**
1113	* Deactivates VT-x on the current CPU.
1114	*
1115	* @returns VBox status code.
1116	* @param pCpu Pointer to the global CPU info struct.
1117	* @param pvCpuPage Pointer to the VMXON region.
1118	* @param HCPhysCpuPage Physical address of the VMXON region.
1119	*
1120	* @remarks This function should never be called when SUPR0EnableVTx() or
1121	* similar was used to enable VT-x on the host.
1122	*/
1123	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1124	{
1125	NOREF(pCpu);
1126	NOREF(pvCpuPage);
1127	NOREF(HCPhysCpuPage);
1128
1129	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1130	return hmR0VmxLeaveRootMode();
1131	}
1132
1133
1134	/**
1135	* Sets the permission bits for the specified MSR in the MSR bitmap.
1136	*
1137	* @param pVCpu Pointer to the VMCPU.
1138	* @param uMSR The MSR value.
1139	* @param enmRead Whether reading this MSR causes a VM-exit.
1140	* @param enmWrite Whether writing this MSR causes a VM-exit.
1141	*/
1142	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1143	{
1144	int32_t iBit;
1145	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1146
1147	/*
1148	* Layout:
1149	* 0x000 - 0x3ff - Low MSR read bits
1150	* 0x400 - 0x7ff - High MSR read bits
1151	* 0x800 - 0xbff - Low MSR write bits
1152	* 0xc00 - 0xfff - High MSR write bits
1153	*/
1154	if (uMsr <= 0x00001FFF)
1155	iBit = uMsr;
1156	else if ( uMsr >= 0xC0000000
1157	&& uMsr <= 0xC0001FFF)
1158	{
1159	iBit = (uMsr - 0xC0000000);
1160	pbMsrBitmap += 0x400;
1161	}
1162	else
1163	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1164
1165	Assert(iBit <= 0x1fff);
1166	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1167	ASMBitSet(pbMsrBitmap, iBit);
1168	else
1169	ASMBitClear(pbMsrBitmap, iBit);
1170
1171	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1172	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1173	else
1174	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1175	}
1176
1177
1178	#ifdef VBOX_STRICT
1179	/**
1180	* Gets the permission bits for the specified MSR in the MSR bitmap.
1181	*
1182	* @returns VBox status code.
1183	* @retval VINF_SUCCESS if the specified MSR is found.
1184	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1185	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1186	*
1187	* @param pVCpu Pointer to the VMCPU.
1188	* @param uMsr The MSR.
1189	* @param penmRead Where to store the read permissions.
1190	* @param penmWrite Where to store the write permissions.
1191	*/
1192	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1193	{
1194	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1195	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1196	int32_t iBit;
1197	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1198
1199	/* See hmR0VmxSetMsrPermission() for the layout. */
1200	if (uMsr <= 0x00001FFF)
1201	iBit = uMsr;
1202	else if ( uMsr >= 0xC0000000
1203	&& uMsr <= 0xC0001FFF)
1204	{
1205	iBit = (uMsr - 0xC0000000);
1206	pbMsrBitmap += 0x400;
1207	}
1208	else
1209	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1210
1211	Assert(iBit <= 0x1fff);
1212	if (ASMBitTest(pbMsrBitmap, iBit))
1213	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1214	else
1215	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1216
1217	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1218	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1219	else
1220	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1221	return VINF_SUCCESS;
1222	}
1223	#endif /* VBOX_STRICT */
1224
1225
1226	/**
1227	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1228	* area.
1229	*
1230	* @returns VBox status code.
1231	* @param pVCpu Pointer to the VMCPU.
1232	* @param cMsrs The number of MSRs.
1233	*/
1234	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1235	{
1236	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1237	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1238	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1239	{
1240	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1241	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1242	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1243	}
1244
1245	/* Update number of guest MSRs to load/store across the world-switch. */
1246	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1247	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1248
1249	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1250	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1251
1252	/* Update the VCPU's copy of the MSR count. */
1253	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1254
1255	return VINF_SUCCESS;
1256	}
1257
1258
1259	/**
1260	* Adds a new (or updates the value of an existing) guest/host MSR
1261	* pair to be swapped during the world-switch as part of the
1262	* auto-load/store MSR area in the VMCS.
1263	*
1264	* @returns true if the MSR was added -and- its value was updated, false
1265	* otherwise.
1266	* @param pVCpu Pointer to the VMCPU.
1267	* @param uMsr The MSR.
1268	* @param uGuestMsr Value of the guest MSR.
1269	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1270	* necessary.
1271	*/
1272	static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1273	{
1274	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1275	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1276	uint32_t i;
1277	for (i = 0; i < cMsrs; i++)
1278	{
1279	if (pGuestMsr->u32Msr == uMsr)
1280	break;
1281	pGuestMsr++;
1282	}
1283
1284	bool fAdded = false;
1285	if (i == cMsrs)
1286	{
1287	++cMsrs;
1288	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1289	AssertRC(rc);
1290
1291	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1292	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1293	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1294
1295	fAdded = true;
1296	}
1297
1298	/* Update the MSR values in the auto-load/store MSR area. */
1299	pGuestMsr->u32Msr = uMsr;
1300	pGuestMsr->u64Value = uGuestMsrValue;
1301
1302	/* Create/update the MSR slot in the host MSR area. */
1303	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1304	pHostMsr += i;
1305	pHostMsr->u32Msr = uMsr;
1306
1307	/*
1308	* Update the host MSR only when requested by the caller AND when we're
1309	* adding it to the auto-load/store area. Otherwise, it would have been
1310	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1311	*/
1312	bool fUpdatedMsrValue = false;
1313	if ( fAdded
1314	&& fUpdateHostMsr)
1315	{
1316	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1317	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1318	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1319	fUpdatedMsrValue = true;
1320	}
1321
1322	return fUpdatedMsrValue;
1323	}
1324
1325
1326	/**
1327	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1328	* auto-load/store MSR area in the VMCS.
1329	*
1330	* @returns VBox status code.
1331	* @param pVCpu Pointer to the VMCPU.
1332	* @param uMsr The MSR.
1333	*/
1334	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1335	{
1336	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1337	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1338	for (uint32_t i = 0; i < cMsrs; i++)
1339	{
1340	/* Find the MSR. */
1341	if (pGuestMsr->u32Msr == uMsr)
1342	{
1343	/* If it's the last MSR, simply reduce the count. */
1344	if (i == cMsrs - 1)
1345	{
1346	--cMsrs;
1347	break;
1348	}
1349
1350	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1351	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1352	pLastGuestMsr += cMsrs - 1;
1353	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1354	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1355
1356	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1357	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1358	pLastHostMsr += cMsrs - 1;
1359	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1360	pHostMsr->u64Value = pLastHostMsr->u64Value;
1361	--cMsrs;
1362	break;
1363	}
1364	pGuestMsr++;
1365	}
1366
1367	/* Update the VMCS if the count changed (meaning the MSR was found). */
1368	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1369	{
1370	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1371	AssertRCReturn(rc, rc);
1372
1373	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1374	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1375	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1376
1377	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1378	return VINF_SUCCESS;
1379	}
1380
1381	return VERR_NOT_FOUND;
1382	}
1383
1384
1385	/**
1386	* Checks if the specified guest MSR is part of the auto-load/store area in
1387	* the VMCS.
1388	*
1389	* @returns true if found, false otherwise.
1390	* @param pVCpu Pointer to the VMCPU.
1391	* @param uMsr The MSR to find.
1392	*/
1393	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1394	{
1395	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1396	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1397
1398	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1399	{
1400	if (pGuestMsr->u32Msr == uMsr)
1401	return true;
1402	}
1403	return false;
1404	}
1405
1406
1407	/**
1408	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1409	*
1410	* @param pVCpu Pointer to the VMCPU.
1411	*
1412	* @remarks No-long-jump zone!!!
1413	*/
1414	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1415	{
1416	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1417	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1418	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1419	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1420
1421	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1422	{
1423	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1424
1425	/*
1426	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1427	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1428	*/
1429	if (pHostMsr->u32Msr == MSR_K6_EFER)
1430	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1431	else
1432	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1433	}
1434
1435	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1436	}
1437
1438
1439	#if HC_ARCH_BITS == 64
1440	/**
1441	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1442	* perform lazy restoration of the host MSRs while leaving VT-x.
1443	*
1444	* @param pVCpu Pointer to the VMCPU.
1445	*
1446	* @remarks No-long-jump zone!!!
1447	*/
1448	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1449	{
1450	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1451
1452	/*
1453	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1454	*/
1455	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1456	{
1457	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1458	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1459	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1460	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1461	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1462	}
1463	}
1464
1465
1466	/**
1467	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1468	* lazily while leaving VT-x.
1469	*
1470	* @returns true if it does, false otherwise.
1471	* @param pVCpu Pointer to the VMCPU.
1472	* @param uMsr The MSR to check.
1473	*/
1474	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1475	{
1476	NOREF(pVCpu);
1477	switch (uMsr)
1478	{
1479	case MSR_K8_LSTAR:
1480	case MSR_K6_STAR:
1481	case MSR_K8_SF_MASK:
1482	case MSR_K8_KERNEL_GS_BASE:
1483	return true;
1484	}
1485	return false;
1486	}
1487
1488
1489	/**
1490	* Saves a set of guest MSRs back into the guest-CPU context.
1491	*
1492	* @param pVCpu Pointer to the VMCPU.
1493	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1494	* out-of-sync. Make sure to update the required fields
1495	* before using them.
1496	*
1497	* @remarks No-long-jump zone!!!
1498	*/
1499	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1500	{
1501	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1502	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1503
1504	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1505	{
1506	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1507	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1508	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1509	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1510	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1511	}
1512	}
1513
1514
1515	/**
1516	* Loads a set of guests MSRs to allow read/passthru to the guest.
1517	*
1518	* The name of this function is slightly confusing. This function does NOT
1519	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1520	* common prefix for functions dealing with "lazy restoration" of the shared
1521	* MSRs.
1522	*
1523	* @param pVCpu Pointer to the VMCPU.
1524	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1525	* out-of-sync. Make sure to update the required fields
1526	* before using them.
1527	*
1528	* @remarks No-long-jump zone!!!
1529	*/
1530	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1531	{
1532	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1536	do { \
1537	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1538	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1539	else \
1540	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1541	} while (0)
1542
1543	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1544	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1545	{
1546	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1547	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1548	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1549	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1550	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1551	}
1552	else
1553	{
1554	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1555	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1556	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1557	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1558	}
1559
1560	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1561	}
1562
1563
1564	/**
1565	* Performs lazy restoration of the set of host MSRs if they were previously
1566	* loaded with guest MSR values.
1567	*
1568	* @param pVCpu Pointer to the VMCPU.
1569	*
1570	* @remarks No-long-jump zone!!!
1571	* @remarks The guest MSRs should have been saved back into the guest-CPU
1572	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1573	*/
1574	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1575	{
1576	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1577	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1578
1579	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1580	{
1581	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1582	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1583	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1584	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1585	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1586	}
1587	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1588	}
1589	#endif /* HC_ARCH_BITS == 64 */
1590
1591
1592	/**
1593	* Verifies that our cached values of the VMCS controls are all
1594	* consistent with what's actually present in the VMCS.
1595	*
1596	* @returns VBox status code.
1597	* @param pVCpu Pointer to the VMCPU.
1598	*/
1599	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1600	{
1601	uint32_t u32Val;
1602	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1603	AssertRCReturn(rc, rc);
1604	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1605	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1606
1607	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1608	AssertRCReturn(rc, rc);
1609	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1610	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1611
1612	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1613	AssertRCReturn(rc, rc);
1614	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1615	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1616
1617	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1618	AssertRCReturn(rc, rc);
1619	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1620	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1621
1622	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1623	{
1624	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1625	AssertRCReturn(rc, rc);
1626	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1627	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1628	}
1629
1630	return VINF_SUCCESS;
1631	}
1632
1633
1634	#ifdef VBOX_STRICT
1635	/**
1636	* Verifies that our cached host EFER value has not changed
1637	* since we cached it.
1638	*
1639	* @param pVCpu Pointer to the VMCPU.
1640	*/
1641	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1642	{
1643	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1644
1645	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1646	{
1647	uint64_t u64Val;
1648	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1649	AssertRC(rc);
1650
1651	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1652	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1653	}
1654	}
1655
1656
1657	/**
1658	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1659	* VMCS are correct.
1660	*
1661	* @param pVCpu Pointer to the VMCPU.
1662	*/
1663	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1664	{
1665	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1666
1667	/* Verify MSR counts in the VMCS are what we think it should be. */
1668	uint32_t cMsrs;
1669	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1670	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1671
1672	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1673	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1674
1675	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1676	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1677
1678	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1679	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1680	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1681	{
1682	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1683	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1684	pGuestMsr->u32Msr, cMsrs));
1685
1686	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1687	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1688	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1689
1690	/* Verify that the permissions are as expected in the MSR bitmap. */
1691	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1692	{
1693	VMXMSREXITREAD enmRead;
1694	VMXMSREXITWRITE enmWrite;
1695	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1696	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1697	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1698	{
1699	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1700	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1701	}
1702	else
1703	{
1704	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1705	pGuestMsr->u32Msr, cMsrs));
1706	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1707	pGuestMsr->u32Msr, cMsrs));
1708	}
1709	}
1710	}
1711	}
1712	#endif /* VBOX_STRICT */
1713
1714
1715	/**
1716	* Flushes the TLB using EPT.
1717	*
1718	* @returns VBox status code.
1719	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1720	* enmFlush).
1721	* @param enmFlush Type of flush.
1722	*
1723	* @remarks Caller is responsible for making sure this function is called only
1724	* when NestedPaging is supported and providing @a enmFlush that is
1725	* supported by the CPU.
1726	* @remarks Can be called with interrupts disabled.
1727	*/
1728	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1729	{
1730	uint64_t au64Descriptor[2];
1731	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1732	au64Descriptor[0] = 0;
1733	else
1734	{
1735	Assert(pVCpu);
1736	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1737	}
1738	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1739
1740	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1741	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1742	rc));
1743	if ( RT_SUCCESS(rc)
1744	&& pVCpu)
1745	{
1746	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1747	}
1748	}
1749
1750
1751	/**
1752	* Flushes the TLB using VPID.
1753	*
1754	* @returns VBox status code.
1755	* @param pVM Pointer to the VM.
1756	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1757	* enmFlush).
1758	* @param enmFlush Type of flush.
1759	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1760	* on @a enmFlush).
1761	*
1762	* @remarks Can be called with interrupts disabled.
1763	*/
1764	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1765	{
1766	NOREF(pVM);
1767	AssertPtr(pVM);
1768	Assert(pVM->hm.s.vmx.fVpid);
1769
1770	uint64_t au64Descriptor[2];
1771	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1772	{
1773	au64Descriptor[0] = 0;
1774	au64Descriptor[1] = 0;
1775	}
1776	else
1777	{
1778	AssertPtr(pVCpu);
1779	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1780	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1781	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1782	au64Descriptor[1] = GCPtr;
1783	}
1784
1785	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1786	AssertMsg(rc == VINF_SUCCESS,
1787	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1788	if ( RT_SUCCESS(rc)
1789	&& pVCpu)
1790	{
1791	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1792	}
1793	}
1794
1795
1796	/**
1797	* Invalidates a guest page by guest virtual address. Only relevant for
1798	* EPT/VPID, otherwise there is nothing really to invalidate.
1799	*
1800	* @returns VBox status code.
1801	* @param pVM Pointer to the VM.
1802	* @param pVCpu Pointer to the VMCPU.
1803	* @param GCVirt Guest virtual address of the page to invalidate.
1804	*/
1805	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1806	{
1807	AssertPtr(pVM);
1808	AssertPtr(pVCpu);
1809	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1810
1811	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1812	if (!fFlushPending)
1813	{
1814	/*
1815	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1816	* See @bugref{6043} and @bugref{6177}.
1817	*
1818	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1819	* function maybe called in a loop with individual addresses.
1820	*/
1821	if (pVM->hm.s.vmx.fVpid)
1822	{
1823	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1824	{
1825	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1826	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1827	}
1828	else
1829	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1830	}
1831	else if (pVM->hm.s.fNestedPaging)
1832	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1833	}
1834
1835	return VINF_SUCCESS;
1836	}
1837
1838
1839	/**
1840	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1841	* otherwise there is nothing really to invalidate.
1842	*
1843	* @returns VBox status code.
1844	* @param pVM Pointer to the VM.
1845	* @param pVCpu Pointer to the VMCPU.
1846	* @param GCPhys Guest physical address of the page to invalidate.
1847	*/
1848	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1849	{
1850	NOREF(pVM); NOREF(GCPhys);
1851	LogFlowFunc(("%RGp\n", GCPhys));
1852
1853	/*
1854	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1855	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1856	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1857	*/
1858	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1859	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1860	return VINF_SUCCESS;
1861	}
1862
1863
1864	/**
1865	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1866	* case where neither EPT nor VPID is supported by the CPU.
1867	*
1868	* @param pVM Pointer to the VM.
1869	* @param pVCpu Pointer to the VMCPU.
1870	* @param pCpu Pointer to the global HM struct.
1871	*
1872	* @remarks Called with interrupts disabled.
1873	*/
1874	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1875	{
1876	AssertPtr(pVCpu);
1877	AssertPtr(pCpu);
1878	NOREF(pVM);
1879
1880	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1881
1882	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1883	#if 0
1884	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1885	pVCpu->hm.s.TlbShootdown.cPages = 0;
1886	#endif
1887
1888	Assert(pCpu->idCpu != NIL_RTCPUID);
1889	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1890	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1891	pVCpu->hm.s.fForceTLBFlush = false;
1892	return;
1893	}
1894
1895
1896	/**
1897	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1898	*
1899	* @param pVM Pointer to the VM.
1900	* @param pVCpu Pointer to the VMCPU.
1901	* @param pCpu Pointer to the global HM CPU struct.
1902	* @remarks All references to "ASID" in this function pertains to "VPID" in
1903	* Intel's nomenclature. The reason is, to avoid confusion in compare
1904	* statements since the host-CPU copies are named "ASID".
1905	*
1906	* @remarks Called with interrupts disabled.
1907	*/
1908	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1909	{
1910	#ifdef VBOX_WITH_STATISTICS
1911	bool fTlbFlushed = false;
1912	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1913	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1914	if (!fTlbFlushed) \
1915	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1916	} while (0)
1917	#else
1918	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1919	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1920	#endif
1921
1922	AssertPtr(pVM);
1923	AssertPtr(pCpu);
1924	AssertPtr(pVCpu);
1925	Assert(pCpu->idCpu != NIL_RTCPUID);
1926
1927	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1928	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1929	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1930
1931	/*
1932	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1933	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1934	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1935	*/
1936	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1937	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1938	{
1939	++pCpu->uCurrentAsid;
1940	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1941	{
1942	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1943	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1944	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1945	}
1946
1947	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1948	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1949	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1950
1951	/*
1952	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1953	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1954	*/
1955	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1956	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1957	HMVMX_SET_TAGGED_TLB_FLUSHED();
1958	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1959	}
1960
1961	/* Check for explicit TLB shootdowns. */
1962	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1963	{
1964	/*
1965	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1966	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1967	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1968	* but not guest-physical mappings.
1969	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1970	*/
1971	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1972	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1973	HMVMX_SET_TAGGED_TLB_FLUSHED();
1974	}
1975
1976	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1977	* where it is commented out. Support individual entry flushing
1978	* someday. */
1979	#if 0
1980	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1981	{
1982	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1983
1984	/*
1985	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1986	* as supported by the CPU.
1987	*/
1988	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1989	{
1990	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1991	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1992	}
1993	else
1994	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1995
1996	HMVMX_SET_TAGGED_TLB_FLUSHED();
1997	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1998	pVCpu->hm.s.TlbShootdown.cPages = 0;
1999	}
2000	#endif
2001
2002	pVCpu->hm.s.fForceTLBFlush = false;
2003
2004	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2005
2006	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2007	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2008	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2009	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2010	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2011	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2012	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2013	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2014	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2015
2016	/* Update VMCS with the VPID. */
2017	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2018	AssertRC(rc);
2019
2020	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2021	}
2022
2023
2024	/**
2025	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2026	*
2027	* @returns VBox status code.
2028	* @param pVM Pointer to the VM.
2029	* @param pVCpu Pointer to the VMCPU.
2030	* @param pCpu Pointer to the global HM CPU struct.
2031	*
2032	* @remarks Called with interrupts disabled.
2033	*/
2034	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2035	{
2036	AssertPtr(pVM);
2037	AssertPtr(pVCpu);
2038	AssertPtr(pCpu);
2039	Assert(pCpu->idCpu != NIL_RTCPUID);
2040	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2041	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2042
2043	/*
2044	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2045	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2046	*/
2047	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2048	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2049	{
2050	pVCpu->hm.s.fForceTLBFlush = true;
2051	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2052	}
2053
2054	/* Check for explicit TLB shootdown flushes. */
2055	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2056	{
2057	pVCpu->hm.s.fForceTLBFlush = true;
2058	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2059	}
2060
2061	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2062	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2063
2064	if (pVCpu->hm.s.fForceTLBFlush)
2065	{
2066	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2067	pVCpu->hm.s.fForceTLBFlush = false;
2068	}
2069	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2070	* where it is commented out. Support individual entry flushing
2071	* someday. */
2072	#if 0
2073	else
2074	{
2075	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2076	{
2077	/* We cannot flush individual entries without VPID support. Flush using EPT. */
2078	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2079	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2080	}
2081	else
2082	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2083
2084	pVCpu->hm.s.TlbShootdown.cPages = 0;
2085	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2086	}
2087	#endif
2088	}
2089
2090
2091	/**
2092	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2093	*
2094	* @returns VBox status code.
2095	* @param pVM Pointer to the VM.
2096	* @param pVCpu Pointer to the VMCPU.
2097	* @param pCpu Pointer to the global HM CPU struct.
2098	*
2099	* @remarks Called with interrupts disabled.
2100	*/
2101	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2102	{
2103	AssertPtr(pVM);
2104	AssertPtr(pVCpu);
2105	AssertPtr(pCpu);
2106	Assert(pCpu->idCpu != NIL_RTCPUID);
2107	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2108	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2109
2110	/*
2111	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2112	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2113	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2114	*/
2115	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2116	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2117	{
2118	pVCpu->hm.s.fForceTLBFlush = true;
2119	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2120	}
2121
2122	/* Check for explicit TLB shootdown flushes. */
2123	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2124	{
2125	/*
2126	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2127	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2128	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2129	*/
2130	pVCpu->hm.s.fForceTLBFlush = true;
2131	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2132	}
2133
2134	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2135	if (pVCpu->hm.s.fForceTLBFlush)
2136	{
2137	++pCpu->uCurrentAsid;
2138	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2139	{
2140	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2141	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2142	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2143	}
2144
2145	pVCpu->hm.s.fForceTLBFlush = false;
2146	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2147	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2148	if (pCpu->fFlushAsidBeforeUse)
2149	{
2150	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2151	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2152	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2153	{
2154	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2155	pCpu->fFlushAsidBeforeUse = false;
2156	}
2157	else
2158	{
2159	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2160	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2161	}
2162	}
2163	}
2164	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2165	* where it is commented out. Support individual entry flushing
2166	* someday. */
2167	#if 0
2168	else
2169	{
2170	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2171	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2172	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2173	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2174
2175	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2176	{
2177	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2178	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2179	{
2180	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2181	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2182	}
2183	else
2184	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2185
2186	pVCpu->hm.s.TlbShootdown.cPages = 0;
2187	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2188	}
2189	else
2190	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2191	}
2192	#endif
2193
2194	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2195	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2196	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2197	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2198	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2199	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2200	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2201
2202	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2203	AssertRC(rc);
2204	}
2205
2206
2207	/**
2208	* Flushes the guest TLB entry based on CPU capabilities.
2209	*
2210	* @param pVCpu Pointer to the VMCPU.
2211	* @param pCpu Pointer to the global HM CPU struct.
2212	*/
2213	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2214	{
2215	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2216	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2217	#endif
2218	PVM pVM = pVCpu->CTX_SUFF(pVM);
2219	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2220	{
2221	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2222	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2223	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2224	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2225	default:
2226	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2227	break;
2228	}
2229
2230	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2231	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2232
2233	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2234	}
2235
2236
2237	/**
2238	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2239	* TLB entries from the host TLB before VM-entry.
2240	*
2241	* @returns VBox status code.
2242	* @param pVM Pointer to the VM.
2243	*/
2244	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2245	{
2246	/*
2247	* Determine optimal flush type for Nested Paging.
2248	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2249	* guest execution (see hmR3InitFinalizeR0()).
2250	*/
2251	if (pVM->hm.s.fNestedPaging)
2252	{
2253	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2254	{
2255	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2256	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2257	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2258	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2259	else
2260	{
2261	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2262	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2263	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2264	}
2265
2266	/* Make sure the write-back cacheable memory type for EPT is supported. */
2267	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2268	{
2269	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2270	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2271	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2272	}
2273	}
2274	else
2275	{
2276	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2277	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2278	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2279	}
2280	}
2281
2282	/*
2283	* Determine optimal flush type for VPID.
2284	*/
2285	if (pVM->hm.s.vmx.fVpid)
2286	{
2287	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2288	{
2289	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2290	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2291	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2292	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2293	else
2294	{
2295	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2296	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2297	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2298	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2299	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2300	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2301	pVM->hm.s.vmx.fVpid = false;
2302	}
2303	}
2304	else
2305	{
2306	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2307	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2308	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2309	pVM->hm.s.vmx.fVpid = false;
2310	}
2311	}
2312
2313	/*
2314	* Setup the handler for flushing tagged-TLBs.
2315	*/
2316	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2317	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2318	else if (pVM->hm.s.fNestedPaging)
2319	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2320	else if (pVM->hm.s.vmx.fVpid)
2321	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2322	else
2323	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2324	return VINF_SUCCESS;
2325	}
2326
2327
2328	/**
2329	* Sets up pin-based VM-execution controls in the VMCS.
2330	*
2331	* @returns VBox status code.
2332	* @param pVM Pointer to the VM.
2333	* @param pVCpu Pointer to the VMCPU.
2334	*/
2335	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2336	{
2337	AssertPtr(pVM);
2338	AssertPtr(pVCpu);
2339
2340	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2341	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2342
2343	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2344	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2345
2346	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2347	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2348
2349	/* Enable the VMX preemption timer. */
2350	if (pVM->hm.s.vmx.fUsePreemptTimer)
2351	{
2352	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2353	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2354	}
2355
2356	if ((val & zap) != val)
2357	{
2358	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2359	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2360	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2361	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2362	}
2363
2364	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2365	AssertRCReturn(rc, rc);
2366
2367	pVCpu->hm.s.vmx.u32PinCtls = val;
2368	return rc;
2369	}
2370
2371
2372	/**
2373	* Sets up processor-based VM-execution controls in the VMCS.
2374	*
2375	* @returns VBox status code.
2376	* @param pVM Pointer to the VM.
2377	* @param pVMCPU Pointer to the VMCPU.
2378	*/
2379	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2380	{
2381	AssertPtr(pVM);
2382	AssertPtr(pVCpu);
2383
2384	int rc = VERR_INTERNAL_ERROR_5;
2385	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2386	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2387
2388	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2389	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2390	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2391	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2392	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2393	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2394	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2395
2396	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2397	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2398	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2399	{
2400	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2401	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2402	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2403	}
2404
2405	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2406	if (!pVM->hm.s.fNestedPaging)
2407	{
2408	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2409	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2410	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2411	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2412	}
2413
2414	/* Use TPR shadowing if supported by the CPU. */
2415	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2416	{
2417	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2418	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2419	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2420	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2421	AssertRCReturn(rc, rc);
2422
2423	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2424	/* CR8 writes cause a VM-exit based on TPR threshold. */
2425	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2426	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2427	}
2428	else
2429	{
2430	/*
2431	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2432	* Set this control only for 64-bit guests.
2433	*/
2434	if (pVM->hm.s.fAllow64BitGuests)
2435	{
2436	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2437	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2438	}
2439	}
2440
2441	/* Use MSR-bitmaps if supported by the CPU. */
2442	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2443	{
2444	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2445
2446	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2447	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2448	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2449	AssertRCReturn(rc, rc);
2450
2451	/*
2452	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2453	* automatically using dedicated fields in the VMCS.
2454	*/
2455	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2456	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2457	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2458	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2459	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2460
2461	#if HC_ARCH_BITS == 64
2462	/*
2463	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2464	*/
2465	if (pVM->hm.s.fAllow64BitGuests)
2466	{
2467	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2468	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2469	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2470	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2471	}
2472	#endif
2473	}
2474
2475	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2476	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2477	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2478
2479	if ((val & zap) != val)
2480	{
2481	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2482	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2483	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2484	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2485	}
2486
2487	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2488	AssertRCReturn(rc, rc);
2489
2490	pVCpu->hm.s.vmx.u32ProcCtls = val;
2491
2492	/*
2493	* Secondary processor-based VM-execution controls.
2494	*/
2495	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2496	{
2497	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2498	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2499
2500	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2501	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2502
2503	if (pVM->hm.s.fNestedPaging)
2504	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2505	else
2506	{
2507	/*
2508	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2509	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2510	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2511	*/
2512	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2513	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2514	}
2515
2516	if (pVM->hm.s.vmx.fVpid)
2517	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2518
2519	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2520	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2521
2522	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2523	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2524	* done dynamically. */
2525	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2526	{
2527	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2528	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2529	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2530	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2531	AssertRCReturn(rc, rc);
2532	}
2533
2534	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2535	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2536
2537	if ((val & zap) != val)
2538	{
2539	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2540	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2541	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2542	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2543	}
2544
2545	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2546	AssertRCReturn(rc, rc);
2547
2548	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2549	}
2550	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2551	{
2552	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2553	"available\n"));
2554	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2555	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2556	}
2557
2558	return VINF_SUCCESS;
2559	}
2560
2561
2562	/**
2563	* Sets up miscellaneous (everything other than Pin & Processor-based
2564	* VM-execution) control fields in the VMCS.
2565	*
2566	* @returns VBox status code.
2567	* @param pVM Pointer to the VM.
2568	* @param pVCpu Pointer to the VMCPU.
2569	*/
2570	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2571	{
2572	NOREF(pVM);
2573	AssertPtr(pVM);
2574	AssertPtr(pVCpu);
2575
2576	int rc = VERR_GENERAL_FAILURE;
2577
2578	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2579	#if 0
2580	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2581	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2582	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2583
2584	/*
2585	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2586	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2587	* We thus use the exception bitmap to control it rather than use both.
2588	*/
2589	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2590	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2591
2592	/** @todo Explore possibility of using IO-bitmaps. */
2593	/* All IO & IOIO instructions cause VM-exits. */
2594	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2595	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2596
2597	/* Initialize the MSR-bitmap area. */
2598	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2599	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2600	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2601	#endif
2602
2603	/* Setup MSR auto-load/store area. */
2604	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2605	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2606	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2607	AssertRCReturn(rc, rc);
2608	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2609	AssertRCReturn(rc, rc);
2610
2611	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2612	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2613	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2614	AssertRCReturn(rc, rc);
2615
2616	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2617	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2618	AssertRCReturn(rc, rc);
2619
2620	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2621	#if 0
2622	/* Setup debug controls */
2623	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2624	AssertRCReturn(rc, rc);
2625	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2626	AssertRCReturn(rc, rc);
2627	#endif
2628
2629	return rc;
2630	}
2631
2632
2633	/**
2634	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2635	*
2636	* @returns VBox status code.
2637	* @param pVM Pointer to the VM.
2638	* @param pVCpu Pointer to the VMCPU.
2639	*/
2640	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2641	{
2642	AssertPtr(pVM);
2643	AssertPtr(pVCpu);
2644
2645	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2646
2647	uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2648
2649	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2650	if (!pVM->hm.s.fNestedPaging)
2651	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2652
2653	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2654	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2655	AssertRCReturn(rc, rc);
2656	return rc;
2657	}
2658
2659
2660	/**
2661	* Sets up the initial guest-state mask. The guest-state mask is consulted
2662	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2663	* for the nested virtualization case (as it would cause a VM-exit).
2664	*
2665	* @param pVCpu Pointer to the VMCPU.
2666	*/
2667	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2668	{
2669	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2670	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2671	return VINF_SUCCESS;
2672	}
2673
2674
2675	/**
2676	* Does per-VM VT-x initialization.
2677	*
2678	* @returns VBox status code.
2679	* @param pVM Pointer to the VM.
2680	*/
2681	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2682	{
2683	LogFlowFunc(("pVM=%p\n", pVM));
2684
2685	int rc = hmR0VmxStructsAlloc(pVM);
2686	if (RT_FAILURE(rc))
2687	{
2688	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2689	return rc;
2690	}
2691
2692	return VINF_SUCCESS;
2693	}
2694
2695
2696	/**
2697	* Does per-VM VT-x termination.
2698	*
2699	* @returns VBox status code.
2700	* @param pVM Pointer to the VM.
2701	*/
2702	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2703	{
2704	LogFlowFunc(("pVM=%p\n", pVM));
2705
2706	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2707	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2708	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2709	#endif
2710	hmR0VmxStructsFree(pVM);
2711	return VINF_SUCCESS;
2712	}
2713
2714
2715	/**
2716	* Sets up the VM for execution under VT-x.
2717	* This function is only called once per-VM during initialization.
2718	*
2719	* @returns VBox status code.
2720	* @param pVM Pointer to the VM.
2721	*/
2722	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2723	{
2724	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2725	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2726
2727	LogFlowFunc(("pVM=%p\n", pVM));
2728
2729	/*
2730	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2731	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2732	*/
2733	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2734	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2735	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2736	{
2737	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2738	return VERR_INTERNAL_ERROR;
2739	}
2740
2741	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2742	/*
2743	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2744	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2745	*/
2746	if ( pVM->hm.s.fAllow64BitGuests
2747	&& !HMVMX_IS_64BIT_HOST_MODE())
2748	{
2749	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2750	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2751	}
2752	#endif
2753
2754	/* Initialize these always, see hmR3InitFinalizeR0().*/
2755	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2756	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2757
2758	/* Setup the tagged-TLB flush handlers. */
2759	int rc = hmR0VmxSetupTaggedTlb(pVM);
2760	if (RT_FAILURE(rc))
2761	{
2762	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2763	return rc;
2764	}
2765
2766	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2767	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2768	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2769	if ( HMVMX_IS_64BIT_HOST_MODE()
2770	&& (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2771	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2772	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2773	{
2774	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2775	}
2776	#endif
2777
2778	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2779	{
2780	PVMCPU pVCpu = &pVM->aCpus[i];
2781	AssertPtr(pVCpu);
2782	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2783
2784	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2785	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2786
2787	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2788	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2789	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2790
2791	/* Set revision dword at the beginning of the VMCS structure. */
2792	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2793
2794	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2795	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2796	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799	/* Load this VMCS as the current VMCS. */
2800	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2801	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2805	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2809	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2810	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2811
2812	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2813	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2814	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2815
2816	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2817	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819
2820	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2821	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2825	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2826	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2827	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2828	#endif
2829
2830	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2831	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2832	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834
2835	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2836
2837	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2838	}
2839
2840	return VINF_SUCCESS;
2841	}
2842
2843
2844	/**
2845	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2846	* the VMCS.
2847	*
2848	* @returns VBox status code.
2849	* @param pVM Pointer to the VM.
2850	* @param pVCpu Pointer to the VMCPU.
2851	*/
2852	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2853	{
2854	NOREF(pVM); NOREF(pVCpu);
2855
2856	RTCCUINTREG uReg = ASMGetCR0();
2857	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2858	AssertRCReturn(rc, rc);
2859
2860	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2861	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2862	if (HMVMX_IS_64BIT_HOST_MODE())
2863	{
2864	uint64_t uRegCR3 = HMR0Get64bitCR3();
2865	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2866	}
2867	else
2868	#endif
2869	{
2870	uReg = ASMGetCR3();
2871	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2872	}
2873	AssertRCReturn(rc, rc);
2874
2875	uReg = ASMGetCR4();
2876	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2877	AssertRCReturn(rc, rc);
2878	return rc;
2879	}
2880
2881
2882	#if HC_ARCH_BITS == 64
2883	/**
2884	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2885	* requirements. See hmR0VmxSaveHostSegmentRegs().
2886	*/
2887	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2888	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2889	{ \
2890	bool fValidSelector = true; \
2891	if ((selValue) & X86_SEL_LDT) \
2892	{ \
2893	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2894	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2895	} \
2896	if (fValidSelector) \
2897	{ \
2898	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2899	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2900	} \
2901	(selValue) = 0; \
2902	}
2903	#endif
2904
2905
2906	/**
2907	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2908	* the host-state area in the VMCS.
2909	*
2910	* @returns VBox status code.
2911	* @param pVM Pointer to the VM.
2912	* @param pVCpu Pointer to the VMCPU.
2913	*/
2914	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2915	{
2916	int rc = VERR_INTERNAL_ERROR_5;
2917
2918	#if HC_ARCH_BITS == 64
2919	/*
2920	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2921	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2922	*/
2923	AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2924	("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2925	#endif
2926
2927	/*
2928	* Host DS, ES, FS and GS segment registers.
2929	*/
2930	#if HC_ARCH_BITS == 64
2931	RTSEL uSelDS = ASMGetDS();
2932	RTSEL uSelES = ASMGetES();
2933	RTSEL uSelFS = ASMGetFS();
2934	RTSEL uSelGS = ASMGetGS();
2935	#else
2936	RTSEL uSelDS = 0;
2937	RTSEL uSelES = 0;
2938	RTSEL uSelFS = 0;
2939	RTSEL uSelGS = 0;
2940	#endif
2941
2942	/* Recalculate which host-state bits need to be manually restored. */
2943	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2944
2945	/*
2946	* Host CS and SS segment registers.
2947	*/
2948	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2949	RTSEL uSelCS;
2950	RTSEL uSelSS;
2951	if (HMVMX_IS_64BIT_HOST_MODE())
2952	{
2953	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2954	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2955	}
2956	else
2957	{
2958	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2959	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2960	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2961	}
2962	#else
2963	RTSEL uSelCS = ASMGetCS();
2964	RTSEL uSelSS = ASMGetSS();
2965	#endif
2966
2967	/*
2968	* Host TR segment register.
2969	*/
2970	RTSEL uSelTR = ASMGetTR();
2971
2972	#if HC_ARCH_BITS == 64
2973	/*
2974	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2975	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2976	*/
2977	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2978	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2979	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2980	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2981	# undef VMXLOCAL_ADJUST_HOST_SEG
2982	#endif
2983
2984	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2985	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2986	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2987	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2988	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2989	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2990	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2991	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2992	Assert(uSelCS);
2993	Assert(uSelTR);
2994
2995	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2996	#if 0
2997	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2998	Assert(uSelSS != 0);
2999	#endif
3000
3001	/* Write these host selector fields into the host-state area in the VMCS. */
3002	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
3003	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
3004	#if HC_ARCH_BITS == 64
3005	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
3006	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
3007	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
3008	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
3009	#endif
3010	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
3011
3012	/*
3013	* Host GDTR and IDTR.
3014	*/
3015	RTGDTR Gdtr;
3016	RT_ZERO(Gdtr);
3017	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3018	if (HMVMX_IS_64BIT_HOST_MODE())
3019	{
3020	X86XDTR64 Gdtr64;
3021	X86XDTR64 Idtr64;
3022	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3023	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3024	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3025
3026	Gdtr.cbGdt = Gdtr64.cb;
3027	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3028	}
3029	else
3030	#endif
3031	{
3032	RTIDTR Idtr;
3033	ASMGetGDTR(&Gdtr);
3034	ASMGetIDTR(&Idtr);
3035	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3036	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3037
3038	#if HC_ARCH_BITS == 64
3039	/*
3040	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3041	* maximum limit (0xffff) on every VM-exit.
3042	*/
3043	if (Gdtr.cbGdt != 0xffff)
3044	{
3045	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3046	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3047	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3048	}
3049
3050	/*
3051	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3052	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3053	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3054	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3055	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3056	* hosts where we are pretty sure it won't cause trouble.
3057	*/
3058	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3059	if (Idtr.cbIdt < 0x0fff)
3060	# else
3061	if (Idtr.cbIdt != 0xffff)
3062	# endif
3063	{
3064	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3065	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3066	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3067	}
3068	#endif
3069	}
3070
3071	/*
3072	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3073	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3074	*/
3075	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3076	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3077	VERR_VMX_INVALID_HOST_STATE);
3078
3079	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3080	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3081	if (HMVMX_IS_64BIT_HOST_MODE())
3082	{
3083	/* We need the 64-bit TR base for hybrid darwin. */
3084	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3085	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3086	}
3087	else
3088	#endif
3089	{
3090	uintptr_t uTRBase;
3091	#if HC_ARCH_BITS == 64
3092	uTRBase = X86DESC64_BASE(pDesc);
3093
3094	/*
3095	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3096	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3097	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3098	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3099	*
3100	* [1] See Intel spec. 3.5 "System Descriptor Types".
3101	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3102	*/
3103	Assert(pDesc->System.u4Type == 11);
3104	if ( pDesc->System.u16LimitLow != 0x67
3105	\|\| pDesc->System.u4LimitHigh)
3106	{
3107	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3108	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3109	if (pVM->hm.s.uHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3110	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3111	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3112
3113	/* Store the GDTR here as we need it while restoring TR. */
3114	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3115	}
3116	#else
3117	uTRBase = X86DESC_BASE(pDesc);
3118	#endif
3119	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3120	}
3121	AssertRCReturn(rc, rc);
3122
3123	/*
3124	* Host FS base and GS base.
3125	*/
3126	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3127	if (HMVMX_IS_64BIT_HOST_MODE())
3128	{
3129	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3130	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3131	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3132	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3133
3134	# if HC_ARCH_BITS == 64
3135	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3136	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3137	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3138	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3139	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3140	# endif
3141	}
3142	#endif
3143	return rc;
3144	}
3145
3146
3147	/**
3148	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3149	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3150	* the host after every successful VM-exit.
3151	*
3152	* @returns VBox status code.
3153	* @param pVM Pointer to the VM.
3154	* @param pVCpu Pointer to the VMCPU.
3155	*
3156	* @remarks No-long-jump zone!!!
3157	*/
3158	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3159	{
3160	NOREF(pVM);
3161
3162	AssertPtr(pVCpu);
3163	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3164
3165	int rc = VINF_SUCCESS;
3166	#if HC_ARCH_BITS == 64
3167	if (pVM->hm.s.fAllow64BitGuests)
3168	hmR0VmxLazySaveHostMsrs(pVCpu);
3169	#endif
3170
3171	/*
3172	* Host Sysenter MSRs.
3173	*/
3174	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3175	AssertRCReturn(rc, rc);
3176	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3177	if (HMVMX_IS_64BIT_HOST_MODE())
3178	{
3179	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3180	AssertRCReturn(rc, rc);
3181	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3182	}
3183	else
3184	{
3185	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3186	AssertRCReturn(rc, rc);
3187	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3188	}
3189	#elif HC_ARCH_BITS == 32
3190	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3191	AssertRCReturn(rc, rc);
3192	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3193	#else
3194	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3195	AssertRCReturn(rc, rc);
3196	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3197	#endif
3198	AssertRCReturn(rc, rc);
3199
3200	/*
3201	* Host EFER MSR.
3202	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3203	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3204	*/
3205	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3206	{
3207	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3208	AssertRCReturn(rc, rc);
3209	}
3210
3211	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3212	* hmR0VmxLoadGuestExitCtls() !! */
3213
3214	return rc;
3215	}
3216
3217
3218	/**
3219	* Figures out if we need to swap the EFER MSR which is
3220	* particularly expensive.
3221	*
3222	* We check all relevant bits. For now, that's everything
3223	* besides LMA/LME, as these two bits are handled by VM-entry,
3224	* see hmR0VmxLoadGuestExitCtls() and
3225	* hmR0VMxLoadGuestEntryCtls().
3226	*
3227	* @returns true if we need to load guest EFER, false otherwise.
3228	* @param pVCpu Pointer to the VMCPU.
3229	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3230	* out-of-sync. Make sure to update the required fields
3231	* before using them.
3232	*
3233	* @remarks Requires EFER, CR4.
3234	* @remarks No-long-jump zone!!!
3235	*/
3236	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3237	{
3238	#ifdef HMVMX_ALWAYS_SWAP_EFER
3239	return true;
3240	#endif
3241
3242	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3243	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3244	if (CPUMIsGuestInLongMode(pVCpu))
3245	return false;
3246	#endif
3247
3248	PVM pVM = pVCpu->CTX_SUFF(pVM);
3249	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3250	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3251
3252	/*
3253	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3254	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3255	*/
3256	if ( CPUMIsGuestInLongMode(pVCpu)
3257	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3258	{
3259	return true;
3260	}
3261
3262	/*
3263	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3264	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3265	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3266	*/
3267	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3268	&& (pMixedCtx->cr0 & X86_CR0_PG)
3269	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3270	{
3271	/* Assert that host is PAE capable. */
3272	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3273	return true;
3274	}
3275
3276	/** @todo Check the latest Intel spec. for any other bits,
3277	* like SMEP/SMAP? */
3278	return false;
3279	}
3280
3281
3282	/**
3283	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3284	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3285	* controls".
3286	*
3287	* @returns VBox status code.
3288	* @param pVCpu Pointer to the VMCPU.
3289	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3290	* out-of-sync. Make sure to update the required fields
3291	* before using them.
3292	*
3293	* @remarks Requires EFER.
3294	* @remarks No-long-jump zone!!!
3295	*/
3296	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3297	{
3298	int rc = VINF_SUCCESS;
3299	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3300	{
3301	PVM pVM = pVCpu->CTX_SUFF(pVM);
3302	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3303	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3304
3305	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3306	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3307
3308	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3309	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3310	{
3311	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3312	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3313	}
3314	else
3315	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3316
3317	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3318	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3319	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3320	{
3321	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3322	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3323	}
3324
3325	/*
3326	* The following should -not- be set (since we're not in SMM mode):
3327	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3328	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3329	*/
3330
3331	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3332	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3333
3334	if ((val & zap) != val)
3335	{
3336	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3337	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3338	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3339	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3340	}
3341
3342	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3343	AssertRCReturn(rc, rc);
3344
3345	pVCpu->hm.s.vmx.u32EntryCtls = val;
3346	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3347	}
3348	return rc;
3349	}
3350
3351
3352	/**
3353	* Sets up the VM-exit controls in the VMCS.
3354	*
3355	* @returns VBox status code.
3356	* @param pVM Pointer to the VM.
3357	* @param pVCpu Pointer to the VMCPU.
3358	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3359	* out-of-sync. Make sure to update the required fields
3360	* before using them.
3361	*
3362	* @remarks Requires EFER.
3363	*/
3364	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3365	{
3366	NOREF(pMixedCtx);
3367
3368	int rc = VINF_SUCCESS;
3369	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3370	{
3371	PVM pVM = pVCpu->CTX_SUFF(pVM);
3372	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3373	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3374
3375	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3376	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3377
3378	/*
3379	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3380	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3381	*/
3382	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3383	if (HMVMX_IS_64BIT_HOST_MODE())
3384	{
3385	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3386	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3387	}
3388	else
3389	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3390	#else
3391	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3392	{
3393	/* The switcher returns to long mode, EFER is managed by the switcher. */
3394	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3395	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3396	}
3397	else
3398	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3399	#endif /* HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3400
3401	/* If the newer VMCS fields for managing EFER exists, use it. */
3402	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3403	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3404	{
3405	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3406	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3407	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3408	}
3409
3410	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3411	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3412
3413	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3414	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3415	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3416
3417	if ( pVM->hm.s.vmx.fUsePreemptTimer
3418	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3419	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3420
3421	if ((val & zap) != val)
3422	{
3423	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3424	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3425	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3426	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3427	}
3428
3429	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3430	AssertRCReturn(rc, rc);
3431
3432	pVCpu->hm.s.vmx.u32ExitCtls = val;
3433	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3434	}
3435	return rc;
3436	}
3437
3438
3439	/**
3440	* Loads the guest APIC and related state.
3441	*
3442	* @returns VBox status code.
3443	* @param pVM Pointer to the VM.
3444	* @param pVCpu Pointer to the VMCPU.
3445	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3446	* out-of-sync. Make sure to update the required fields
3447	* before using them.
3448	*/
3449	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3450	{
3451	NOREF(pMixedCtx);
3452
3453	int rc = VINF_SUCCESS;
3454	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3455	{
3456	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3457	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3458	{
3459	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3460
3461	bool fPendingIntr = false;
3462	uint8_t u8Tpr = 0;
3463	uint8_t u8PendingIntr = 0;
3464	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3465	AssertRCReturn(rc, rc);
3466
3467	/*
3468	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3469	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3470	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3471	* the interrupt when we VM-exit for other reasons.
3472	*/
3473	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3474	uint32_t u32TprThreshold = 0;
3475	if (fPendingIntr)
3476	{
3477	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3478	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3479	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3480	if (u8PendingPriority <= u8TprPriority)
3481	u32TprThreshold = u8PendingPriority;
3482	else
3483	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3484	}
3485	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3486
3487	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3488	AssertRCReturn(rc, rc);
3489	}
3490
3491	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3492	}
3493	return rc;
3494	}
3495
3496
3497	/**
3498	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3499	*
3500	* @returns Guest's interruptibility-state.
3501	* @param pVCpu Pointer to the VMCPU.
3502	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3503	* out-of-sync. Make sure to update the required fields
3504	* before using them.
3505	*
3506	* @remarks No-long-jump zone!!!
3507	*/
3508	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3509	{
3510	/*
3511	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3512	*/
3513	uint32_t uIntrState = 0;
3514	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3515	{
3516	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3517	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3518	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3519	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3520	{
3521	if (pMixedCtx->eflags.Bits.u1IF)
3522	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3523	else
3524	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3525	}
3526	/* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3527	}
3528
3529	/*
3530	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3531	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3532	* setting this would block host-NMIs and IRET will not clear the blocking.
3533	*
3534	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3535	*/
3536	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3537	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3538	{
3539	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3540	}
3541
3542	return uIntrState;
3543	}
3544
3545
3546	/**
3547	* Loads the guest's interruptibility-state into the guest-state area in the
3548	* VMCS.
3549	*
3550	* @returns VBox status code.
3551	* @param pVCpu Pointer to the VMCPU.
3552	* @param uIntrState The interruptibility-state to set.
3553	*/
3554	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3555	{
3556	NOREF(pVCpu);
3557	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3558	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3559	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3560	AssertRCReturn(rc, rc);
3561	return rc;
3562	}
3563
3564
3565	/**
3566	* Loads the exception intercepts required for guest execution in the VMCS.
3567	*
3568	* @returns VBox status code.
3569	* @param pVCpu Pointer to the VMCPU.
3570	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3571	* out-of-sync. Make sure to update the required fields
3572	* before using them.
3573	*/
3574	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3575	{
3576	NOREF(pMixedCtx);
3577	int rc = VINF_SUCCESS;
3578	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3579	{
3580	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3581	if (pVCpu->hm.s.fGIMTrapXcptUD)
3582	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3583	else
3584	{
3585	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3586	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3587	#endif
3588	}
3589
3590	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3591	AssertRCReturn(rc, rc);
3592
3593	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3594	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3595	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3596	}
3597	return rc;
3598	}
3599
3600
3601	/**
3602	* Loads the guest's RIP into the guest-state area in the VMCS.
3603	*
3604	* @returns VBox status code.
3605	* @param pVCpu Pointer to the VMCPU.
3606	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3607	* out-of-sync. Make sure to update the required fields
3608	* before using them.
3609	*
3610	* @remarks No-long-jump zone!!!
3611	*/
3612	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3613	{
3614	int rc = VINF_SUCCESS;
3615	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3616	{
3617	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3618	AssertRCReturn(rc, rc);
3619
3620	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3621	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3622	HMCPU_CF_VALUE(pVCpu)));
3623	}
3624	return rc;
3625	}
3626
3627
3628	/**
3629	* Loads the guest's RSP into the guest-state area in the VMCS.
3630	*
3631	* @returns VBox status code.
3632	* @param pVCpu Pointer to the VMCPU.
3633	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3634	* out-of-sync. Make sure to update the required fields
3635	* before using them.
3636	*
3637	* @remarks No-long-jump zone!!!
3638	*/
3639	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3640	{
3641	int rc = VINF_SUCCESS;
3642	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3643	{
3644	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3645	AssertRCReturn(rc, rc);
3646
3647	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3648	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3649	}
3650	return rc;
3651	}
3652
3653
3654	/**
3655	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3656	*
3657	* @returns VBox status code.
3658	* @param pVCpu Pointer to the VMCPU.
3659	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3660	* out-of-sync. Make sure to update the required fields
3661	* before using them.
3662	*
3663	* @remarks No-long-jump zone!!!
3664	*/
3665	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3666	{
3667	int rc = VINF_SUCCESS;
3668	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3669	{
3670	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3671	Let us assert it as such and use 32-bit VMWRITE. */
3672	Assert(!(pMixedCtx->rflags.u64 >> 32));
3673	X86EFLAGS Eflags = pMixedCtx->eflags;
3674	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3675	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3676	* These will never be cleared/set, unless some other part of the VMM
3677	* code is buggy - in which case we're better of finding and fixing
3678	* those bugs than hiding them. */
3679	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3680	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3681	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3682	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3683
3684	/*
3685	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3686	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3687	*/
3688	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3689	{
3690	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3691	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3692	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3693	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3694	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3695	}
3696
3697	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3698	AssertRCReturn(rc, rc);
3699
3700	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3701	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3702	}
3703	return rc;
3704	}
3705
3706
3707	/**
3708	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3709	*
3710	* @returns VBox status code.
3711	* @param pVCpu Pointer to the VMCPU.
3712	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3713	* out-of-sync. Make sure to update the required fields
3714	* before using them.
3715	*
3716	* @remarks No-long-jump zone!!!
3717	*/
3718	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3719	{
3720	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3721	AssertRCReturn(rc, rc);
3722	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3723	AssertRCReturn(rc, rc);
3724	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3725	AssertRCReturn(rc, rc);
3726	return rc;
3727	}
3728
3729
3730	/**
3731	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3732	* CR0 is partially shared with the host and we have to consider the FPU bits.
3733	*
3734	* @returns VBox status code.
3735	* @param pVM Pointer to the VM.
3736	* @param pVCpu Pointer to the VMCPU.
3737	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3738	* out-of-sync. Make sure to update the required fields
3739	* before using them.
3740	*
3741	* @remarks No-long-jump zone!!!
3742	*/
3743	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3744	{
3745	/*
3746	* Guest CR0.
3747	* Guest FPU.
3748	*/
3749	int rc = VINF_SUCCESS;
3750	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3751	{
3752	Assert(!(pMixedCtx->cr0 >> 32));
3753	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3754	PVM pVM = pVCpu->CTX_SUFF(pVM);
3755
3756	/* The guest's view (read access) of its CR0 is unblemished. */
3757	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3758	AssertRCReturn(rc, rc);
3759	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3760
3761	/* Setup VT-x's view of the guest CR0. */
3762	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3763	if (pVM->hm.s.fNestedPaging)
3764	{
3765	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3766	{
3767	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3768	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3769	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3770	}
3771	else
3772	{
3773	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3774	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3775	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3776	}
3777
3778	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3779	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3780	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3781
3782	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3783	AssertRCReturn(rc, rc);
3784	}
3785	else
3786	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3787
3788	/*
3789	* Guest FPU bits.
3790	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3791	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3792	*/
3793	u32GuestCR0 \|= X86_CR0_NE;
3794	bool fInterceptNM = false;
3795	if (CPUMIsGuestFPUStateActive(pVCpu))
3796	{
3797	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3798	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3799	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3800	}
3801	else
3802	{
3803	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3804	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3805	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3806	}
3807
3808	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3809	bool fInterceptMF = false;
3810	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3811	fInterceptMF = true;
3812
3813	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3814	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3815	{
3816	Assert(PDMVmmDevHeapIsEnabled(pVM));
3817	Assert(pVM->hm.s.vmx.pRealModeTSS);
3818	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3819	fInterceptNM = true;
3820	fInterceptMF = true;
3821	}
3822	else
3823	{
3824	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626} comment #11. */
3825	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3826	}
3827	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3828
3829	if (fInterceptNM)
3830	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3831	else
3832	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3833
3834	if (fInterceptMF)
3835	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3836	else
3837	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3838
3839	/* Additional intercepts for debugging, define these yourself explicitly. */
3840	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3841	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3842	\| RT_BIT(X86_XCPT_BP)
3843	\| RT_BIT(X86_XCPT_DB)
3844	\| RT_BIT(X86_XCPT_DE)
3845	\| RT_BIT(X86_XCPT_NM)
3846	\| RT_BIT(X86_XCPT_TS)
3847	\| RT_BIT(X86_XCPT_UD)
3848	\| RT_BIT(X86_XCPT_NP)
3849	\| RT_BIT(X86_XCPT_SS)
3850	\| RT_BIT(X86_XCPT_GP)
3851	\| RT_BIT(X86_XCPT_PF)
3852	\| RT_BIT(X86_XCPT_MF)
3853	;
3854	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3855	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3856	#endif
3857
3858	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3859
3860	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3861	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3862	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3863	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3864	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3865	else
3866	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3867
3868	u32GuestCR0 \|= uSetCR0;
3869	u32GuestCR0 &= uZapCR0;
3870	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3871
3872	/* Write VT-x's view of the guest CR0 into the VMCS. */
3873	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3874	AssertRCReturn(rc, rc);
3875	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3876	uZapCR0));
3877
3878	/*
3879	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3880	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3881	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3882	*/
3883	uint32_t u32CR0Mask = 0;
3884	u32CR0Mask = X86_CR0_PE
3885	\| X86_CR0_NE
3886	\| X86_CR0_WP
3887	\| X86_CR0_PG
3888	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3889	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3890	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3891
3892	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3893	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3894	* and @bugref{6944}. */
3895	#if 0
3896	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3897	u32CR0Mask &= ~X86_CR0_PE;
3898	#endif
3899	if (pVM->hm.s.fNestedPaging)
3900	u32CR0Mask &= ~X86_CR0_WP;
3901
3902	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3903	if (fInterceptNM)
3904	{
3905	u32CR0Mask \|= X86_CR0_TS
3906	\| X86_CR0_MP;
3907	}
3908
3909	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3910	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3911	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3912	AssertRCReturn(rc, rc);
3913	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3914
3915	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3916	}
3917	return rc;
3918	}
3919
3920
3921	/**
3922	* Loads the guest control registers (CR3, CR4) into the guest-state area
3923	* in the VMCS.
3924	*
3925	* @returns VBox status code.
3926	* @param pVM Pointer to the VM.
3927	* @param pVCpu Pointer to the VMCPU.
3928	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3929	* out-of-sync. Make sure to update the required fields
3930	* before using them.
3931	*
3932	* @remarks No-long-jump zone!!!
3933	*/
3934	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3935	{
3936	int rc = VINF_SUCCESS;
3937	PVM pVM = pVCpu->CTX_SUFF(pVM);
3938
3939	/*
3940	* Guest CR2.
3941	* It's always loaded in the assembler code. Nothing to do here.
3942	*/
3943
3944	/*
3945	* Guest CR3.
3946	*/
3947	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3948	{
3949	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3950	if (pVM->hm.s.fNestedPaging)
3951	{
3952	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3953
3954	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3955	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3956	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3957	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3958
3959	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3960	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3961	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3962
3963	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3964	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3965	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3966	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3967
3968	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3969	AssertRCReturn(rc, rc);
3970	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3971
3972	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3973	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3974	{
3975	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3976	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3977	{
3978	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3979	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3980	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3981	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3982	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3983	}
3984
3985	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3986	have Unrestricted Execution to handle the guest when it's not using paging. */
3987	GCPhysGuestCR3 = pMixedCtx->cr3;
3988	}
3989	else
3990	{
3991	/*
3992	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3993	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3994	* EPT takes care of translating it to host-physical addresses.
3995	*/
3996	RTGCPHYS GCPhys;
3997	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3998	Assert(PDMVmmDevHeapIsEnabled(pVM));
3999
4000	/* We obtain it here every time as the guest could have relocated this PCI region. */
4001	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4002	AssertRCReturn(rc, rc);
4003
4004	GCPhysGuestCR3 = GCPhys;
4005	}
4006
4007	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4008	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4009	}
4010	else
4011	{
4012	/* Non-nested paging case, just use the hypervisor's CR3. */
4013	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4014
4015	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4016	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4017	}
4018	AssertRCReturn(rc, rc);
4019
4020	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4021	}
4022
4023	/*
4024	* Guest CR4.
4025	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4026	*/
4027	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4028	{
4029	Assert(!(pMixedCtx->cr4 >> 32));
4030	uint32_t u32GuestCR4 = pMixedCtx->cr4;
4031
4032	/* The guest's view of its CR4 is unblemished. */
4033	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4034	AssertRCReturn(rc, rc);
4035	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4036
4037	/* Setup VT-x's view of the guest CR4. */
4038	/*
4039	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4040	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4041	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4042	*/
4043	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4044	{
4045	Assert(pVM->hm.s.vmx.pRealModeTSS);
4046	Assert(PDMVmmDevHeapIsEnabled(pVM));
4047	u32GuestCR4 &= ~X86_CR4_VME;
4048	}
4049
4050	if (pVM->hm.s.fNestedPaging)
4051	{
4052	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4053	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4054	{
4055	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4056	u32GuestCR4 \|= X86_CR4_PSE;
4057	/* Our identity mapping is a 32-bit page directory. */
4058	u32GuestCR4 &= ~X86_CR4_PAE;
4059	}
4060	/* else use guest CR4.*/
4061	}
4062	else
4063	{
4064	/*
4065	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4066	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4067	*/
4068	switch (pVCpu->hm.s.enmShadowMode)
4069	{
4070	case PGMMODE_REAL: /* Real-mode. */
4071	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4072	case PGMMODE_32_BIT: /* 32-bit paging. */
4073	{
4074	u32GuestCR4 &= ~X86_CR4_PAE;
4075	break;
4076	}
4077
4078	case PGMMODE_PAE: /* PAE paging. */
4079	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4080	{
4081	u32GuestCR4 \|= X86_CR4_PAE;
4082	break;
4083	}
4084
4085	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4086	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4087	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4088	break;
4089	#endif
4090	default:
4091	AssertFailed();
4092	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4093	}
4094	}
4095
4096	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4097	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4098	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4099	u32GuestCR4 \|= uSetCR4;
4100	u32GuestCR4 &= uZapCR4;
4101
4102	/* Write VT-x's view of the guest CR4 into the VMCS. */
4103	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4104	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4105	AssertRCReturn(rc, rc);
4106
4107	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4108	uint32_t u32CR4Mask = X86_CR4_VME
4109	\| X86_CR4_PAE
4110	\| X86_CR4_PGE
4111	\| X86_CR4_PSE
4112	\| X86_CR4_VMXE;
4113	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4114	u32CR4Mask \|= X86_CR4_OSXSAVE;
4115	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4116	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4117	AssertRCReturn(rc, rc);
4118
4119	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4120	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4121
4122	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4123	}
4124	return rc;
4125	}
4126
4127
4128	/**
4129	* Loads the guest debug registers into the guest-state area in the VMCS.
4130	* This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4131	*
4132	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4133	*
4134	* @returns VBox status code.
4135	* @param pVCpu Pointer to the VMCPU.
4136	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4137	* out-of-sync. Make sure to update the required fields
4138	* before using them.
4139	*
4140	* @remarks No-long-jump zone!!!
4141	*/
4142	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4143	{
4144	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4145	return VINF_SUCCESS;
4146
4147	#ifdef VBOX_STRICT
4148	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4149	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4150	{
4151	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4152	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4153	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4154	}
4155	#endif
4156
4157	int rc;
4158	PVM pVM = pVCpu->CTX_SUFF(pVM);
4159	bool fInterceptDB = false;
4160	bool fInterceptMovDRx = false;
4161	if ( pVCpu->hm.s.fSingleInstruction
4162	\|\| DBGFIsStepping(pVCpu))
4163	{
4164	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4165	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4166	{
4167	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4168	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4169	AssertRCReturn(rc, rc);
4170	Assert(fInterceptDB == false);
4171	}
4172	else
4173	{
4174	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4175	pVCpu->hm.s.fClearTrapFlag = true;
4176	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4177	fInterceptDB = true;
4178	}
4179	}
4180
4181	if ( fInterceptDB
4182	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4183	{
4184	/*
4185	* Use the combined guest and host DRx values found in the hypervisor
4186	* register set because the debugger has breakpoints active or someone
4187	* is single stepping on the host side without a monitor trap flag.
4188	*
4189	* Note! DBGF expects a clean DR6 state before executing guest code.
4190	*/
4191	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4192	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4193	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4194	{
4195	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4196	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4197	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4198	}
4199	else
4200	#endif
4201	if (!CPUMIsHyperDebugStateActive(pVCpu))
4202	{
4203	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4204	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4205	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4206	}
4207
4208	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4209	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4210	AssertRCReturn(rc, rc);
4211
4212	pVCpu->hm.s.fUsingHyperDR7 = true;
4213	fInterceptDB = true;
4214	fInterceptMovDRx = true;
4215	}
4216	else
4217	{
4218	/*
4219	* If the guest has enabled debug registers, we need to load them prior to
4220	* executing guest code so they'll trigger at the right time.
4221	*/
4222	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4223	{
4224	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4225	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4226	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4227	{
4228	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4229	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4230	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4231	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4232	}
4233	else
4234	#endif
4235	if (!CPUMIsGuestDebugStateActive(pVCpu))
4236	{
4237	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4238	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4239	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4240	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4241	}
4242	Assert(!fInterceptDB);
4243	Assert(!fInterceptMovDRx);
4244	}
4245	/*
4246	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4247	* must intercept #DB in order to maintain a correct DR6 guest value.
4248	*/
4249	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4250	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4251	&& !CPUMIsGuestDebugStateActive(pVCpu))
4252	#else
4253	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4254	#endif
4255	{
4256	fInterceptMovDRx = true;
4257	fInterceptDB = true;
4258	}
4259
4260	/* Update guest DR7. */
4261	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4262	AssertRCReturn(rc, rc);
4263
4264	pVCpu->hm.s.fUsingHyperDR7 = false;
4265	}
4266
4267	/*
4268	* Update the exception bitmap regarding intercepting #DB generated by the guest.
4269	*/
4270	if ( fInterceptDB
4271	\|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4272	{
4273	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
4274	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
4275	}
4276	else
4277	{
4278	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4279	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4280	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
4281	#endif
4282	}
4283
4284	/*
4285	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4286	*/
4287	if (fInterceptMovDRx)
4288	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4289	else
4290	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4291	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4292	AssertRCReturn(rc, rc);
4293
4294	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4295	return VINF_SUCCESS;
4296	}
4297
4298
4299	#ifdef VBOX_STRICT
4300	/**
4301	* Strict function to validate segment registers.
4302	*
4303	* @remarks ASSUMES CR0 is up to date.
4304	*/
4305	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4306	{
4307	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4308	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4309	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4310	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4311	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4312	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4313	{
4314	/* Protected mode checks */
4315	/* CS */
4316	Assert(pCtx->cs.Attr.n.u1Present);
4317	Assert(!(pCtx->cs.Attr.u & 0xf00));
4318	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4319	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4320	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4321	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4322	\|\| (pCtx->cs.Attr.n.u1Granularity));
4323	/* CS cannot be loaded with NULL in protected mode. */
4324	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4325	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4326	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4327	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4328	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4329	else
4330	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4331	/* SS */
4332	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4333	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4334	if ( !(pCtx->cr0 & X86_CR0_PE)
4335	\|\| pCtx->cs.Attr.n.u4Type == 3)
4336	{
4337	Assert(!pCtx->ss.Attr.n.u2Dpl);
4338	}
4339	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4340	{
4341	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4342	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4343	Assert(pCtx->ss.Attr.n.u1Present);
4344	Assert(!(pCtx->ss.Attr.u & 0xf00));
4345	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4346	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4347	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4348	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4349	\|\| (pCtx->ss.Attr.n.u1Granularity));
4350	}
4351	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4352	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4353	{
4354	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4355	Assert(pCtx->ds.Attr.n.u1Present);
4356	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4357	Assert(!(pCtx->ds.Attr.u & 0xf00));
4358	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4359	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4360	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4361	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4362	\|\| (pCtx->ds.Attr.n.u1Granularity));
4363	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4364	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4365	}
4366	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4367	{
4368	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4369	Assert(pCtx->es.Attr.n.u1Present);
4370	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4371	Assert(!(pCtx->es.Attr.u & 0xf00));
4372	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4373	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4374	\|\| !(pCtx->es.Attr.n.u1Granularity));
4375	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4376	\|\| (pCtx->es.Attr.n.u1Granularity));
4377	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4378	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4379	}
4380	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4381	{
4382	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4383	Assert(pCtx->fs.Attr.n.u1Present);
4384	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4385	Assert(!(pCtx->fs.Attr.u & 0xf00));
4386	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4387	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4388	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4389	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4390	\|\| (pCtx->fs.Attr.n.u1Granularity));
4391	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4392	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4393	}
4394	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4395	{
4396	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4397	Assert(pCtx->gs.Attr.n.u1Present);
4398	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4399	Assert(!(pCtx->gs.Attr.u & 0xf00));
4400	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4401	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4402	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4403	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4404	\|\| (pCtx->gs.Attr.n.u1Granularity));
4405	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4406	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4407	}
4408	/* 64-bit capable CPUs. */
4409	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4410	if (HMVMX_IS_64BIT_HOST_MODE())
4411	{
4412	Assert(!(pCtx->cs.u64Base >> 32));
4413	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4414	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4415	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4416	}
4417	# endif
4418	}
4419	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4420	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4421	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4422	{
4423	/* Real and v86 mode checks. */
4424	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4425	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4426	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4427	{
4428	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4429	}
4430	else
4431	{
4432	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4433	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4434	}
4435
4436	/* CS */
4437	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4438	Assert(pCtx->cs.u32Limit == 0xffff);
4439	Assert(u32CSAttr == 0xf3);
4440	/* SS */
4441	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4442	Assert(pCtx->ss.u32Limit == 0xffff);
4443	Assert(u32SSAttr == 0xf3);
4444	/* DS */
4445	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4446	Assert(pCtx->ds.u32Limit == 0xffff);
4447	Assert(u32DSAttr == 0xf3);
4448	/* ES */
4449	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4450	Assert(pCtx->es.u32Limit == 0xffff);
4451	Assert(u32ESAttr == 0xf3);
4452	/* FS */
4453	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4454	Assert(pCtx->fs.u32Limit == 0xffff);
4455	Assert(u32FSAttr == 0xf3);
4456	/* GS */
4457	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4458	Assert(pCtx->gs.u32Limit == 0xffff);
4459	Assert(u32GSAttr == 0xf3);
4460	/* 64-bit capable CPUs. */
4461	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4462	if (HMVMX_IS_64BIT_HOST_MODE())
4463	{
4464	Assert(!(pCtx->cs.u64Base >> 32));
4465	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4466	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4467	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4468	}
4469	# endif
4470	}
4471	}
4472	#endif /* VBOX_STRICT */
4473
4474
4475	/**
4476	* Writes a guest segment register into the guest-state area in the VMCS.
4477	*
4478	* @returns VBox status code.
4479	* @param pVCpu Pointer to the VMCPU.
4480	* @param idxSel Index of the selector in the VMCS.
4481	* @param idxLimit Index of the segment limit in the VMCS.
4482	* @param idxBase Index of the segment base in the VMCS.
4483	* @param idxAccess Index of the access rights of the segment in the VMCS.
4484	* @param pSelReg Pointer to the segment selector.
4485	*
4486	* @remarks No-long-jump zone!!!
4487	*/
4488	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4489	uint32_t idxAccess, PCPUMSELREG pSelReg)
4490	{
4491	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4492	AssertRCReturn(rc, rc);
4493	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4494	AssertRCReturn(rc, rc);
4495	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4496	AssertRCReturn(rc, rc);
4497
4498	uint32_t u32Access = pSelReg->Attr.u;
4499	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4500	{
4501	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4502	u32Access = 0xf3;
4503	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4504	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4505	}
4506	else
4507	{
4508	/*
4509	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4510	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4511	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4512	* loaded in protected-mode have their attribute as 0.
4513	*/
4514	if (!u32Access)
4515	u32Access = X86DESCATTR_UNUSABLE;
4516	}
4517
4518	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4519	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4520	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4521
4522	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4523	AssertRCReturn(rc, rc);
4524	return rc;
4525	}
4526
4527
4528	/**
4529	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4530	* into the guest-state area in the VMCS.
4531	*
4532	* @returns VBox status code.
4533	* @param pVM Pointer to the VM.
4534	* @param pVCPU Pointer to the VMCPU.
4535	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4536	* out-of-sync. Make sure to update the required fields
4537	* before using them.
4538	*
4539	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4540	* @remarks No-long-jump zone!!!
4541	*/
4542	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4543	{
4544	int rc = VERR_INTERNAL_ERROR_5;
4545	PVM pVM = pVCpu->CTX_SUFF(pVM);
4546
4547	/*
4548	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4549	*/
4550	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4551	{
4552	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4553	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4554	{
4555	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4556	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4557	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4558	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4559	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4560	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4561	}
4562
4563	#ifdef VBOX_WITH_REM
4564	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4565	{
4566	Assert(pVM->hm.s.vmx.pRealModeTSS);
4567	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4568	if ( pVCpu->hm.s.vmx.fWasInRealMode
4569	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4570	{
4571	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4572	in real-mode (e.g. OpenBSD 4.0) */
4573	REMFlushTBs(pVM);
4574	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4575	pVCpu->hm.s.vmx.fWasInRealMode = false;
4576	}
4577	}
4578	#endif
4579	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4580	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4581	AssertRCReturn(rc, rc);
4582	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4583	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4584	AssertRCReturn(rc, rc);
4585	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4586	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4587	AssertRCReturn(rc, rc);
4588	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4589	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4590	AssertRCReturn(rc, rc);
4591	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4592	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4593	AssertRCReturn(rc, rc);
4594	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4595	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4596	AssertRCReturn(rc, rc);
4597
4598	#ifdef VBOX_STRICT
4599	/* Validate. */
4600	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4601	#endif
4602
4603	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4604	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4605	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4606	}
4607
4608	/*
4609	* Guest TR.
4610	*/
4611	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4612	{
4613	/*
4614	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4615	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4616	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4617	*/
4618	uint16_t u16Sel = 0;
4619	uint32_t u32Limit = 0;
4620	uint64_t u64Base = 0;
4621	uint32_t u32AccessRights = 0;
4622
4623	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4624	{
4625	u16Sel = pMixedCtx->tr.Sel;
4626	u32Limit = pMixedCtx->tr.u32Limit;
4627	u64Base = pMixedCtx->tr.u64Base;
4628	u32AccessRights = pMixedCtx->tr.Attr.u;
4629	}
4630	else
4631	{
4632	Assert(pVM->hm.s.vmx.pRealModeTSS);
4633	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4634
4635	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4636	RTGCPHYS GCPhys;
4637	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4638	AssertRCReturn(rc, rc);
4639
4640	X86DESCATTR DescAttr;
4641	DescAttr.u = 0;
4642	DescAttr.n.u1Present = 1;
4643	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4644
4645	u16Sel = 0;
4646	u32Limit = HM_VTX_TSS_SIZE;
4647	u64Base = GCPhys; /* in real-mode phys = virt. */
4648	u32AccessRights = DescAttr.u;
4649	}
4650
4651	/* Validate. */
4652	Assert(!(u16Sel & RT_BIT(2)));
4653	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4654	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4655	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4656	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4657	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4658	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4659	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4660	Assert( (u32Limit & 0xfff) == 0xfff
4661	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4662	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4663	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4664
4665	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4666	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4667	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4668	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4669
4670	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4671	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4672	}
4673
4674	/*
4675	* Guest GDTR.
4676	*/
4677	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4678	{
4679	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4680	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4681
4682	/* Validate. */
4683	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4684
4685	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4686	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4687	}
4688
4689	/*
4690	* Guest LDTR.
4691	*/
4692	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4693	{
4694	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4695	uint32_t u32Access = 0;
4696	if (!pMixedCtx->ldtr.Attr.u)
4697	u32Access = X86DESCATTR_UNUSABLE;
4698	else
4699	u32Access = pMixedCtx->ldtr.Attr.u;
4700
4701	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4702	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4703	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4704	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4705
4706	/* Validate. */
4707	if (!(u32Access & X86DESCATTR_UNUSABLE))
4708	{
4709	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4710	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4711	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4712	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4713	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4714	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4715	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4716	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4717	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4718	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4719	}
4720
4721	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4722	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4723	}
4724
4725	/*
4726	* Guest IDTR.
4727	*/
4728	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4729	{
4730	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4731	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4732
4733	/* Validate. */
4734	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4735
4736	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4737	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4738	}
4739
4740	return VINF_SUCCESS;
4741	}
4742
4743
4744	/**
4745	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4746	* areas.
4747	*
4748	* These MSRs will automatically be loaded to the host CPU on every successful
4749	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4750	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4751	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4752	*
4753	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4754	*
4755	* @returns VBox status code.
4756	* @param pVCpu Pointer to the VMCPU.
4757	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4758	* out-of-sync. Make sure to update the required fields
4759	* before using them.
4760	*
4761	* @remarks No-long-jump zone!!!
4762	*/
4763	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4764	{
4765	AssertPtr(pVCpu);
4766	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4767
4768	/*
4769	* MSRs that we use the auto-load/store MSR area in the VMCS.
4770	*/
4771	PVM pVM = pVCpu->CTX_SUFF(pVM);
4772	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4773	{
4774	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4775	#if HC_ARCH_BITS == 32 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4776	if (pVM->hm.s.fAllow64BitGuests)
4777	{
4778	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4779	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4780	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4781	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4782	# ifdef DEBUG
4783	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4784	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4785	{
4786	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4787	pMsr->u64Value));
4788	}
4789	# endif
4790	}
4791	#endif
4792	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4793	}
4794
4795	/*
4796	* Guest Sysenter MSRs.
4797	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4798	* VM-exits on WRMSRs for these MSRs.
4799	*/
4800	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4801	{
4802	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4803	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4804	}
4805
4806	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4807	{
4808	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4809	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4810	}
4811
4812	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4813	{
4814	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4815	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4816	}
4817
4818	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4819	{
4820	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4821	{
4822	/*
4823	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4824	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4825	*/
4826	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4827	{
4828	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4829	AssertRCReturn(rc,rc);
4830	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4831	}
4832	else
4833	{
4834	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4835	/* We need to intercept reads too, see @bugref{7386} comment #16. */
4836	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4837	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4838	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4839	}
4840	}
4841	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4842	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4843	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4844	}
4845
4846	return VINF_SUCCESS;
4847	}
4848
4849
4850	/**
4851	* Loads the guest activity state into the guest-state area in the VMCS.
4852	*
4853	* @returns VBox status code.
4854	* @param pVCpu Pointer to the VMCPU.
4855	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4856	* out-of-sync. Make sure to update the required fields
4857	* before using them.
4858	*
4859	* @remarks No-long-jump zone!!!
4860	*/
4861	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4862	{
4863	NOREF(pCtx);
4864	/** @todo See if we can make use of other states, e.g.
4865	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4866	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4867	{
4868	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4869	AssertRCReturn(rc, rc);
4870
4871	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4872	}
4873	return VINF_SUCCESS;
4874	}
4875
4876
4877	/**
4878	* Sets up the appropriate function to run guest code.
4879	*
4880	* @returns VBox status code.
4881	* @param pVCpu Pointer to the VMCPU.
4882	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4883	* out-of-sync. Make sure to update the required fields
4884	* before using them.
4885	*
4886	* @remarks No-long-jump zone!!!
4887	*/
4888	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4889	{
4890	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4891	{
4892	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4893	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4894	#endif
4895	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4896	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4897	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4898	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4899	{
4900	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4901	{
4902	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4903	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4904	\| HM_CHANGED_VMX_ENTRY_CTLS
4905	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4906	}
4907	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4908	}
4909	#else
4910	/* 64-bit host or hybrid host. */
4911	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4912	#endif
4913	}
4914	else
4915	{
4916	/* Guest is not in long mode, use the 32-bit handler. */
4917	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4918	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4919	{
4920	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4921	{
4922	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4923	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4924	\| HM_CHANGED_VMX_ENTRY_CTLS
4925	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4926	}
4927	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4928	}
4929	#else
4930	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4931	#endif
4932	}
4933	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4934	return VINF_SUCCESS;
4935	}
4936
4937
4938	/**
4939	* Wrapper for running the guest code in VT-x.
4940	*
4941	* @returns VBox strict status code.
4942	* @param pVM Pointer to the VM.
4943	* @param pVCpu Pointer to the VMCPU.
4944	* @param pCtx Pointer to the guest-CPU context.
4945	*
4946	* @remarks No-long-jump zone!!!
4947	*/
4948	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4949	{
4950	/*
4951	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4952	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4953	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4954	*/
4955	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4956	/** @todo Add stats for resume vs launch. */
4957	#ifdef VBOX_WITH_KERNEL_USING_XMM
4958	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4959	#else
4960	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4961	#endif
4962	}
4963
4964
4965	/**
4966	* Reports world-switch error and dumps some useful debug info.
4967	*
4968	* @param pVM Pointer to the VM.
4969	* @param pVCpu Pointer to the VMCPU.
4970	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4971	* @param pCtx Pointer to the guest-CPU context.
4972	* @param pVmxTransient Pointer to the VMX transient structure (only
4973	* exitReason updated).
4974	*/
4975	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4976	{
4977	Assert(pVM);
4978	Assert(pVCpu);
4979	Assert(pCtx);
4980	Assert(pVmxTransient);
4981	HMVMX_ASSERT_PREEMPT_SAFE();
4982
4983	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4984	switch (rcVMRun)
4985	{
4986	case VERR_VMX_INVALID_VMXON_PTR:
4987	AssertFailed();
4988	break;
4989	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4990	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4991	{
4992	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4993	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4994	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4995	AssertRC(rc);
4996
4997	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4998	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4999	Cannot do it here as we may have been long preempted. */
5000
5001	#ifdef VBOX_STRICT
5002	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5003	pVmxTransient->uExitReason));
5004	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5005	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5006	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5007	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5008	else
5009	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5010	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5011	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5012
5013	/* VMX control bits. */
5014	uint32_t u32Val;
5015	uint64_t u64Val;
5016	HMVMXHCUINTREG uHCReg;
5017	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5018	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5019	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5020	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5021	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5022	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5023	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5024	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5025	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5026	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5027	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5028	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5029	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5030	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5031	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5032	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5033	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5034	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5035	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5036	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5037	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5038	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5039	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5040	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5041	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5042	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5043	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5044	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5045	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5046	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5047	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5048	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5049	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5050	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5051	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5052	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5053	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5054	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5055	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5056	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5057	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5058	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5059
5060	/* Guest bits. */
5061	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5062	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5063	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5064	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5065	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5066	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5067	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5068	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5069
5070	/* Host bits. */
5071	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5072	Log4(("Host CR0 %#RHr\n", uHCReg));
5073	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5074	Log4(("Host CR3 %#RHr\n", uHCReg));
5075	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5076	Log4(("Host CR4 %#RHr\n", uHCReg));
5077
5078	RTGDTR HostGdtr;
5079	PCX86DESCHC pDesc;
5080	ASMGetGDTR(&HostGdtr);
5081	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5082	Log4(("Host CS %#08x\n", u32Val));
5083	if (u32Val < HostGdtr.cbGdt)
5084	{
5085	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5086	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5087	}
5088
5089	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5090	Log4(("Host DS %#08x\n", u32Val));
5091	if (u32Val < HostGdtr.cbGdt)
5092	{
5093	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5094	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5095	}
5096
5097	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5098	Log4(("Host ES %#08x\n", u32Val));
5099	if (u32Val < HostGdtr.cbGdt)
5100	{
5101	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5102	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5103	}
5104
5105	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5106	Log4(("Host FS %#08x\n", u32Val));
5107	if (u32Val < HostGdtr.cbGdt)
5108	{
5109	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5110	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5111	}
5112
5113	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5114	Log4(("Host GS %#08x\n", u32Val));
5115	if (u32Val < HostGdtr.cbGdt)
5116	{
5117	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5118	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5119	}
5120
5121	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5122	Log4(("Host SS %#08x\n", u32Val));
5123	if (u32Val < HostGdtr.cbGdt)
5124	{
5125	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5126	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5127	}
5128
5129	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5130	Log4(("Host TR %#08x\n", u32Val));
5131	if (u32Val < HostGdtr.cbGdt)
5132	{
5133	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5134	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5135	}
5136
5137	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5138	Log4(("Host TR Base %#RHv\n", uHCReg));
5139	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5140	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5141	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5142	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5143	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5144	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5145	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5146	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5147	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5148	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5149	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5150	Log4(("Host RSP %#RHv\n", uHCReg));
5151	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5152	Log4(("Host RIP %#RHv\n", uHCReg));
5153	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5154	if (HMVMX_IS_64BIT_HOST_MODE())
5155	{
5156	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5157	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5158	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5159	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5160	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5161	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5162	}
5163	# endif
5164	#endif /* VBOX_STRICT */
5165	break;
5166	}
5167
5168	default:
5169	/* Impossible */
5170	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5171	break;
5172	}
5173	NOREF(pVM); NOREF(pCtx);
5174	}
5175
5176
5177	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5178	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5179	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5180	#endif
5181	#ifdef VBOX_STRICT
5182	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5183	{
5184	switch (idxField)
5185	{
5186	case VMX_VMCS_GUEST_RIP:
5187	case VMX_VMCS_GUEST_RSP:
5188	case VMX_VMCS_GUEST_SYSENTER_EIP:
5189	case VMX_VMCS_GUEST_SYSENTER_ESP:
5190	case VMX_VMCS_GUEST_GDTR_BASE:
5191	case VMX_VMCS_GUEST_IDTR_BASE:
5192	case VMX_VMCS_GUEST_CS_BASE:
5193	case VMX_VMCS_GUEST_DS_BASE:
5194	case VMX_VMCS_GUEST_ES_BASE:
5195	case VMX_VMCS_GUEST_FS_BASE:
5196	case VMX_VMCS_GUEST_GS_BASE:
5197	case VMX_VMCS_GUEST_SS_BASE:
5198	case VMX_VMCS_GUEST_LDTR_BASE:
5199	case VMX_VMCS_GUEST_TR_BASE:
5200	case VMX_VMCS_GUEST_CR3:
5201	return true;
5202	}
5203	return false;
5204	}
5205
5206	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5207	{
5208	switch (idxField)
5209	{
5210	/* Read-only fields. */
5211	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5212	return true;
5213	}
5214	/* Remaining readable fields should also be writable. */
5215	return hmR0VmxIsValidWriteField(idxField);
5216	}
5217	#endif /* VBOX_STRICT */
5218
5219
5220	/**
5221	* Executes the specified handler in 64-bit mode.
5222	*
5223	* @returns VBox status code.
5224	* @param pVM Pointer to the VM.
5225	* @param pVCpu Pointer to the VMCPU.
5226	* @param pCtx Pointer to the guest CPU context.
5227	* @param enmOp The operation to perform.
5228	* @param cParams Number of parameters.
5229	* @param paParam Array of 32-bit parameters.
5230	*/
5231	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5232	uint32_t cParams, uint32_t *paParam)
5233	{
5234	int rc, rc2;
5235	PHMGLOBALCPUINFO pCpu;
5236	RTHCPHYS HCPhysCpuPage;
5237	RTCCUINTREG uOldEflags;
5238
5239	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5240	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5241	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5242	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5243
5244	#ifdef VBOX_STRICT
5245	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5246	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5247
5248	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5249	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5250	#endif
5251
5252	/* Disable interrupts. */
5253	uOldEflags = ASMIntDisableFlags();
5254
5255	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5256	RTCPUID idHostCpu = RTMpCpuId();
5257	CPUMR0SetLApic(pVCpu, idHostCpu);
5258	#endif
5259
5260	pCpu = HMR0GetCurrentCpu();
5261	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5262
5263	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5264	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5265
5266	/* Leave VMX Root Mode. */
5267	VMXDisable();
5268
5269	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5270
5271	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5272	CPUMSetHyperEIP(pVCpu, enmOp);
5273	for (int i = (int)cParams - 1; i >= 0; i--)
5274	CPUMPushHyper(pVCpu, paParam[i]);
5275
5276	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5277
5278	/* Call the switcher. */
5279	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5280	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5281
5282	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5283	/* Make sure the VMX instructions don't cause #UD faults. */
5284	SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5285
5286	/* Re-enter VMX Root Mode */
5287	rc2 = VMXEnable(HCPhysCpuPage);
5288	if (RT_FAILURE(rc2))
5289	{
5290	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5291	ASMSetFlags(uOldEflags);
5292	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5293	return rc2;
5294	}
5295
5296	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5297	AssertRC(rc2);
5298	Assert(!(ASMGetFlags() & X86_EFL_IF));
5299	ASMSetFlags(uOldEflags);
5300	return rc;
5301	}
5302
5303
5304	/**
5305	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5306	* supporting 64-bit guests.
5307	*
5308	* @returns VBox status code.
5309	* @param fResume Whether to VMLAUNCH or VMRESUME.
5310	* @param pCtx Pointer to the guest-CPU context.
5311	* @param pCache Pointer to the VMCS cache.
5312	* @param pVM Pointer to the VM.
5313	* @param pVCpu Pointer to the VMCPU.
5314	*/
5315	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5316	{
5317	PHMGLOBALCPUINFO pCpu = NULL;
5318	RTHCPHYS HCPhysCpuPage = 0;
5319	int rc = VERR_INTERNAL_ERROR_5;
5320
5321	pCpu = HMR0GetCurrentCpu();
5322	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5323
5324	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5325	pCache->uPos = 1;
5326	pCache->interPD = PGMGetInterPaeCR3(pVM);
5327	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5328	#endif
5329
5330	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5331	pCache->TestIn.HCPhysCpuPage = 0;
5332	pCache->TestIn.HCPhysVmcs = 0;
5333	pCache->TestIn.pCache = 0;
5334	pCache->TestOut.HCPhysVmcs = 0;
5335	pCache->TestOut.pCache = 0;
5336	pCache->TestOut.pCtx = 0;
5337	pCache->TestOut.eflags = 0;
5338	#endif
5339
5340	uint32_t aParam[10];
5341	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5342	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5343	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5344	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5345	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5346	aParam[5] = 0;
5347	aParam[6] = VM_RC_ADDR(pVM, pVM);
5348	aParam[7] = 0;
5349	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5350	aParam[9] = 0;
5351
5352	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5353	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5354	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5355	#endif
5356	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5357
5358	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5359	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5360	Assert(pCtx->dr[4] == 10);
5361	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5362	#endif
5363
5364	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5365	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5366	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5367	pVCpu->hm.s.vmx.HCPhysVmcs));
5368	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5369	pCache->TestOut.HCPhysVmcs));
5370	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5371	pCache->TestOut.pCache));
5372	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5373	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5374	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5375	pCache->TestOut.pCtx));
5376	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5377	#endif
5378	return rc;
5379	}
5380
5381
5382	/**
5383	* Initialize the VMCS-Read cache.
5384	*
5385	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5386	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5387	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5388	* (those that have a 32-bit FULL & HIGH part).
5389	*
5390	* @returns VBox status code.
5391	* @param pVM Pointer to the VM.
5392	* @param pVCpu Pointer to the VMCPU.
5393	*/
5394	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5395	{
5396	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5397	{ \
5398	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5399	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5400	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5401	++cReadFields; \
5402	}
5403
5404	AssertPtr(pVM);
5405	AssertPtr(pVCpu);
5406	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5407	uint32_t cReadFields = 0;
5408
5409	/*
5410	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5411	* and serve to indicate exceptions to the rules.
5412	*/
5413
5414	/* Guest-natural selector base fields. */
5415	#if 0
5416	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5417	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5418	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5419	#endif
5420	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5421	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5422	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5423	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5424	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5425	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5426	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5427	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5428	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5429	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5430	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5431	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5432	#if 0
5433	/* Unused natural width guest-state fields. */
5434	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5435	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5436	#endif
5437	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5438	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5439
5440	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5441	#if 0
5442	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5443	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5444	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5445	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5446	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5447	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5448	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5449	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5450	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5451	#endif
5452
5453	/* Natural width guest-state fields. */
5454	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5455	#if 0
5456	/* Currently unused field. */
5457	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5458	#endif
5459
5460	if (pVM->hm.s.fNestedPaging)
5461	{
5462	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5463	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5464	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5465	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5466	}
5467	else
5468	{
5469	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5470	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5471	}
5472
5473	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5474	return VINF_SUCCESS;
5475	}
5476
5477
5478	/**
5479	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5480	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5481	* darwin, running 64-bit guests).
5482	*
5483	* @returns VBox status code.
5484	* @param pVCpu Pointer to the VMCPU.
5485	* @param idxField The VMCS field encoding.
5486	* @param u64Val 16, 32 or 64-bit value.
5487	*/
5488	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5489	{
5490	int rc;
5491	switch (idxField)
5492	{
5493	/*
5494	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5495	*/
5496	/* 64-bit Control fields. */
5497	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5498	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5499	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5500	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5501	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5502	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5503	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5504	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5505	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5506	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5507	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5508	case VMX_VMCS64_CTRL_EPTP_FULL:
5509	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5510	/* 64-bit Guest-state fields. */
5511	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5512	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5513	case VMX_VMCS64_GUEST_PAT_FULL:
5514	case VMX_VMCS64_GUEST_EFER_FULL:
5515	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5516	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5517	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5518	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5519	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5520	/* 64-bit Host-state fields. */
5521	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5522	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5523	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5524	{
5525	rc = VMXWriteVmcs32(idxField, u64Val);
5526	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5527	break;
5528	}
5529
5530	/*
5531	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5532	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5533	*/
5534	/* Natural-width Guest-state fields. */
5535	case VMX_VMCS_GUEST_CR3:
5536	case VMX_VMCS_GUEST_ES_BASE:
5537	case VMX_VMCS_GUEST_CS_BASE:
5538	case VMX_VMCS_GUEST_SS_BASE:
5539	case VMX_VMCS_GUEST_DS_BASE:
5540	case VMX_VMCS_GUEST_FS_BASE:
5541	case VMX_VMCS_GUEST_GS_BASE:
5542	case VMX_VMCS_GUEST_LDTR_BASE:
5543	case VMX_VMCS_GUEST_TR_BASE:
5544	case VMX_VMCS_GUEST_GDTR_BASE:
5545	case VMX_VMCS_GUEST_IDTR_BASE:
5546	case VMX_VMCS_GUEST_RSP:
5547	case VMX_VMCS_GUEST_RIP:
5548	case VMX_VMCS_GUEST_SYSENTER_ESP:
5549	case VMX_VMCS_GUEST_SYSENTER_EIP:
5550	{
5551	if (!(u64Val >> 32))
5552	{
5553	/* If this field is 64-bit, VT-x will zero out the top bits. */
5554	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5555	}
5556	else
5557	{
5558	/* Assert that only the 32->64 switcher case should ever come here. */
5559	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5560	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5561	}
5562	break;
5563	}
5564
5565	default:
5566	{
5567	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5568	rc = VERR_INVALID_PARAMETER;
5569	break;
5570	}
5571	}
5572	AssertRCReturn(rc, rc);
5573	return rc;
5574	}
5575
5576
5577	/**
5578	* Queue up a VMWRITE by using the VMCS write cache.
5579	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5580	*
5581	* @param pVCpu Pointer to the VMCPU.
5582	* @param idxField The VMCS field encoding.
5583	* @param u64Val 16, 32 or 64-bit value.
5584	*/
5585	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5586	{
5587	AssertPtr(pVCpu);
5588	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5589
5590	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5591	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5592
5593	/* Make sure there are no duplicates. */
5594	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5595	{
5596	if (pCache->Write.aField[i] == idxField)
5597	{
5598	pCache->Write.aFieldVal[i] = u64Val;
5599	return VINF_SUCCESS;
5600	}
5601	}
5602
5603	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5604	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5605	pCache->Write.cValidEntries++;
5606	return VINF_SUCCESS;
5607	}
5608
5609	/* Enable later when the assembly code uses these as callbacks. */
5610	#if 0
5611	/*
5612	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5613	*
5614	* @param pVCpu Pointer to the VMCPU.
5615	* @param pCache Pointer to the VMCS cache.
5616	*
5617	* @remarks No-long-jump zone!!!
5618	*/
5619	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5620	{
5621	AssertPtr(pCache);
5622	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5623	{
5624	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5625	AssertRC(rc);
5626	}
5627	pCache->Write.cValidEntries = 0;
5628	}
5629
5630
5631	/**
5632	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5633	*
5634	* @param pVCpu Pointer to the VMCPU.
5635	* @param pCache Pointer to the VMCS cache.
5636	*
5637	* @remarks No-long-jump zone!!!
5638	*/
5639	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5640	{
5641	AssertPtr(pCache);
5642	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5643	{
5644	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5645	AssertRC(rc);
5646	}
5647	}
5648	#endif
5649	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5650
5651
5652	/**
5653	* Sets up the usage of TSC-offsetting and updates the VMCS.
5654	*
5655	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5656	* VMX preemption timer.
5657	*
5658	* @returns VBox status code.
5659	* @param pVM Pointer to the cross context VM structure.
5660	* @param pVCpu Pointer to the VMCPU.
5661	*
5662	* @remarks No-long-jump zone!!!
5663	*/
5664	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5665	{
5666	int rc;
5667	bool fOffsettedTsc;
5668	bool fParavirtTsc;
5669	if (pVM->hm.s.vmx.fUsePreemptTimer)
5670	{
5671	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5672	&fOffsettedTsc, &fParavirtTsc);
5673
5674	/* Make sure the returned values have sane upper and lower boundaries. */
5675	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5676	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5677	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5678	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5679
5680	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5681	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5682	}
5683	else
5684	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5685
5686	/** @todo later optimize this to be done elsewhere and not before every
5687	* VM-entry. */
5688	if (fParavirtTsc)
5689	{
5690	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5691	AssertRC(rc);
5692	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5693	}
5694
5695	if (fOffsettedTsc)
5696	{
5697	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5698	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5699
5700	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5701	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5702	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5703	}
5704	else
5705	{
5706	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5707	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5708	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5709	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5710	}
5711	}
5712
5713
5714	/**
5715	* Determines if an exception is a contributory exception.
5716	*
5717	* Contributory exceptions are ones which can cause double-faults unless the
5718	* original exception was a benign exception. Page-fault is intentionally not
5719	* included here as it's a conditional contributory exception.
5720	*
5721	* @returns true if the exception is contributory, false otherwise.
5722	* @param uVector The exception vector.
5723	*/
5724	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5725	{
5726	switch (uVector)
5727	{
5728	case X86_XCPT_GP:
5729	case X86_XCPT_SS:
5730	case X86_XCPT_NP:
5731	case X86_XCPT_TS:
5732	case X86_XCPT_DE:
5733	return true;
5734	default:
5735	break;
5736	}
5737	return false;
5738	}
5739
5740
5741	/**
5742	* Sets an event as a pending event to be injected into the guest.
5743	*
5744	* @param pVCpu Pointer to the VMCPU.
5745	* @param u32IntInfo The VM-entry interruption-information field.
5746	* @param cbInstr The VM-entry instruction length in bytes (for software
5747	* interrupts, exceptions and privileged software
5748	* exceptions).
5749	* @param u32ErrCode The VM-entry exception error code.
5750	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5751	* page-fault.
5752	*
5753	* @remarks Statistics counter assumes this is a guest event being injected or
5754	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5755	* always incremented.
5756	*/
5757	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5758	RTGCUINTPTR GCPtrFaultAddress)
5759	{
5760	Assert(!pVCpu->hm.s.Event.fPending);
5761	pVCpu->hm.s.Event.fPending = true;
5762	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5763	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5764	pVCpu->hm.s.Event.cbInstr = cbInstr;
5765	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5766
5767	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5768	}
5769
5770
5771	/**
5772	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5773	*
5774	* @param pVCpu Pointer to the VMCPU.
5775	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5776	* out-of-sync. Make sure to update the required fields
5777	* before using them.
5778	*/
5779	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5780	{
5781	NOREF(pMixedCtx);
5782	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5783	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5784	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5785	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5786	}
5787
5788
5789	/**
5790	* Handle a condition that occurred while delivering an event through the guest
5791	* IDT.
5792	*
5793	* @returns VBox status code (informational error codes included).
5794	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5795	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5796	* continue execution of the guest which will delivery the #DF.
5797	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5798	*
5799	* @param pVCpu Pointer to the VMCPU.
5800	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5801	* out-of-sync. Make sure to update the required fields
5802	* before using them.
5803	* @param pVmxTransient Pointer to the VMX transient structure.
5804	*
5805	* @remarks No-long-jump zone!!!
5806	*/
5807	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5808	{
5809	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5810
5811	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5812	AssertRCReturn(rc, rc);
5813	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5814	AssertRCReturn(rc, rc);
5815
5816	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5817	{
5818	uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5819	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5820
5821	typedef enum
5822	{
5823	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5824	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5825	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5826	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5827	} VMXREFLECTXCPT;
5828
5829	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5830	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5831	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5832	{
5833	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5834	{
5835	enmReflect = VMXREFLECTXCPT_XCPT;
5836	#ifdef VBOX_STRICT
5837	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5838	&& uExitVector == X86_XCPT_PF)
5839	{
5840	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5841	}
5842	#endif
5843	if ( uExitVector == X86_XCPT_PF
5844	&& uIdtVector == X86_XCPT_PF)
5845	{
5846	pVmxTransient->fVectoringDoublePF = true;
5847	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5848	}
5849	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5850	&& hmR0VmxIsContributoryXcpt(uExitVector)
5851	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5852	\|\| uIdtVector == X86_XCPT_PF))
5853	{
5854	enmReflect = VMXREFLECTXCPT_DF;
5855	}
5856	else if (uIdtVector == X86_XCPT_DF)
5857	enmReflect = VMXREFLECTXCPT_TF;
5858	}
5859	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5860	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5861	{
5862	/*
5863	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5864	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5865	*/
5866	enmReflect = VMXREFLECTXCPT_XCPT;
5867
5868	if (uExitVector == X86_XCPT_PF)
5869	{
5870	pVmxTransient->fVectoringPF = true;
5871	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5872	}
5873	}
5874	}
5875	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5876	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5877	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5878	{
5879	/*
5880	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5881	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5882	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5883	*/
5884	enmReflect = VMXREFLECTXCPT_XCPT;
5885	}
5886
5887	/*
5888	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5889	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5890	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5891	*
5892	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5893	*/
5894	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5895	&& enmReflect == VMXREFLECTXCPT_XCPT
5896	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5897	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5898	{
5899	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5900	}
5901
5902	switch (enmReflect)
5903	{
5904	case VMXREFLECTXCPT_XCPT:
5905	{
5906	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5907	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5908	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5909
5910	uint32_t u32ErrCode = 0;
5911	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5912	{
5913	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5914	AssertRCReturn(rc, rc);
5915	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5916	}
5917
5918	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5919	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5920	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5921	rc = VINF_SUCCESS;
5922	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5923	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5924
5925	break;
5926	}
5927
5928	case VMXREFLECTXCPT_DF:
5929	{
5930	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5931	rc = VINF_HM_DOUBLE_FAULT;
5932	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5933	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5934
5935	break;
5936	}
5937
5938	case VMXREFLECTXCPT_TF:
5939	{
5940	rc = VINF_EM_RESET;
5941	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5942	uExitVector));
5943	break;
5944	}
5945
5946	default:
5947	Assert(rc == VINF_SUCCESS);
5948	break;
5949	}
5950	}
5951	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5952	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5953	&& uExitVector != X86_XCPT_DF
5954	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5955	{
5956	/*
5957	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5958	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5959	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5960	*/
5961	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5962	{
5963	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5964	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5965	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5966	}
5967	}
5968
5969	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5970	return rc;
5971	}
5972
5973
5974	/**
5975	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5976	*
5977	* @returns VBox status code.
5978	* @param pVCpu Pointer to the VMCPU.
5979	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5980	* out-of-sync. Make sure to update the required fields
5981	* before using them.
5982	*
5983	* @remarks No-long-jump zone!!!
5984	*/
5985	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5986	{
5987	NOREF(pMixedCtx);
5988
5989	/*
5990	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5991	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5992	*/
5993	VMMRZCallRing3Disable(pVCpu);
5994	HM_DISABLE_PREEMPT();
5995
5996	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5997	{
5998	uint32_t uVal = 0;
5999	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6000	AssertRCReturn(rc, rc);
6001
6002	uint32_t uShadow = 0;
6003	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6004	AssertRCReturn(rc, rc);
6005
6006	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6007	CPUMSetGuestCR0(pVCpu, uVal);
6008	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6009	}
6010
6011	HM_RESTORE_PREEMPT();
6012	VMMRZCallRing3Enable(pVCpu);
6013	return VINF_SUCCESS;
6014	}
6015
6016
6017	/**
6018	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6019	*
6020	* @returns VBox status code.
6021	* @param pVCpu Pointer to the VMCPU.
6022	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6023	* out-of-sync. Make sure to update the required fields
6024	* before using them.
6025	*
6026	* @remarks No-long-jump zone!!!
6027	*/
6028	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6029	{
6030	NOREF(pMixedCtx);
6031
6032	int rc = VINF_SUCCESS;
6033	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6034	{
6035	uint32_t uVal = 0;
6036	uint32_t uShadow = 0;
6037	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6038	AssertRCReturn(rc, rc);
6039	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6040	AssertRCReturn(rc, rc);
6041
6042	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6043	CPUMSetGuestCR4(pVCpu, uVal);
6044	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6045	}
6046	return rc;
6047	}
6048
6049
6050	/**
6051	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
6052	*
6053	* @returns VBox status code.
6054	* @param pVCpu Pointer to the VMCPU.
6055	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6056	* out-of-sync. Make sure to update the required fields
6057	* before using them.
6058	*
6059	* @remarks No-long-jump zone!!!
6060	*/
6061	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6062	{
6063	int rc = VINF_SUCCESS;
6064	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6065	{
6066	uint64_t u64Val = 0;
6067	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6068	AssertRCReturn(rc, rc);
6069
6070	pMixedCtx->rip = u64Val;
6071	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6072	}
6073	return rc;
6074	}
6075
6076
6077	/**
6078	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
6079	*
6080	* @returns VBox status code.
6081	* @param pVCpu Pointer to the VMCPU.
6082	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6083	* out-of-sync. Make sure to update the required fields
6084	* before using them.
6085	*
6086	* @remarks No-long-jump zone!!!
6087	*/
6088	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6089	{
6090	int rc = VINF_SUCCESS;
6091	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6092	{
6093	uint64_t u64Val = 0;
6094	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6095	AssertRCReturn(rc, rc);
6096
6097	pMixedCtx->rsp = u64Val;
6098	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6099	}
6100	return rc;
6101	}
6102
6103
6104	/**
6105	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6106	*
6107	* @returns VBox status code.
6108	* @param pVCpu Pointer to the VMCPU.
6109	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6110	* out-of-sync. Make sure to update the required fields
6111	* before using them.
6112	*
6113	* @remarks No-long-jump zone!!!
6114	*/
6115	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6116	{
6117	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6118	{
6119	uint32_t uVal = 0;
6120	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6121	AssertRCReturn(rc, rc);
6122
6123	pMixedCtx->eflags.u32 = uVal;
6124	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6125	{
6126	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6127	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6128
6129	pMixedCtx->eflags.Bits.u1VM = 0;
6130	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6131	}
6132
6133	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6134	}
6135	return VINF_SUCCESS;
6136	}
6137
6138
6139	/**
6140	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6141	* guest-CPU context.
6142	*/
6143	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6144	{
6145	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6146	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6147	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6148	return rc;
6149	}
6150
6151
6152	/**
6153	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6154	* from the guest-state area in the VMCS.
6155	*
6156	* @param pVCpu Pointer to the VMCPU.
6157	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6158	* out-of-sync. Make sure to update the required fields
6159	* before using them.
6160	*
6161	* @remarks No-long-jump zone!!!
6162	*/
6163	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6164	{
6165	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6166	{
6167	uint32_t uIntrState = 0;
6168	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6169	AssertRC(rc);
6170
6171	if (!uIntrState)
6172	{
6173	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6174	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6175
6176	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6177	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6178	}
6179	else
6180	{
6181	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6182	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6183	{
6184	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6185	AssertRC(rc);
6186	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6187	AssertRC(rc);
6188
6189	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6190	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6191	}
6192	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6193	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6194
6195	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6196	{
6197	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6198	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6199	}
6200	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6201	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6202	}
6203
6204	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6205	}
6206	}
6207
6208
6209	/**
6210	* Saves the guest's activity state.
6211	*
6212	* @returns VBox status code.
6213	* @param pVCpu Pointer to the VMCPU.
6214	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6215	* out-of-sync. Make sure to update the required fields
6216	* before using them.
6217	*
6218	* @remarks No-long-jump zone!!!
6219	*/
6220	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6221	{
6222	NOREF(pMixedCtx);
6223	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6224	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6225	return VINF_SUCCESS;
6226	}
6227
6228
6229	/**
6230	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6231	* the current VMCS into the guest-CPU context.
6232	*
6233	* @returns VBox status code.
6234	* @param pVCpu Pointer to the VMCPU.
6235	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6236	* out-of-sync. Make sure to update the required fields
6237	* before using them.
6238	*
6239	* @remarks No-long-jump zone!!!
6240	*/
6241	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6242	{
6243	int rc = VINF_SUCCESS;
6244	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6245	{
6246	uint32_t u32Val = 0;
6247	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6248	pMixedCtx->SysEnter.cs = u32Val;
6249	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6250	}
6251
6252	uint64_t u64Val = 0;
6253	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6254	{
6255	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6256	pMixedCtx->SysEnter.eip = u64Val;
6257	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6258	}
6259	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6260	{
6261	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6262	pMixedCtx->SysEnter.esp = u64Val;
6263	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6264	}
6265	return rc;
6266	}
6267
6268
6269	/**
6270	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6271	* the CPU back into the guest-CPU context.
6272	*
6273	* @returns VBox status code.
6274	* @param pVCpu Pointer to the VMCPU.
6275	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6276	* out-of-sync. Make sure to update the required fields
6277	* before using them.
6278	*
6279	* @remarks No-long-jump zone!!!
6280	*/
6281	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6282	{
6283	#if HC_ARCH_BITS == 64
6284	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6285	{
6286	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6287	VMMRZCallRing3Disable(pVCpu);
6288	HM_DISABLE_PREEMPT();
6289
6290	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6291	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6292	{
6293	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6294	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6295	}
6296
6297	HM_RESTORE_PREEMPT();
6298	VMMRZCallRing3Enable(pVCpu);
6299	}
6300	else
6301	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6302	#else
6303	NOREF(pMixedCtx);
6304	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6305	#endif
6306
6307	return VINF_SUCCESS;
6308	}
6309
6310
6311	/**
6312	* Saves the auto load/store'd guest MSRs from the current VMCS into
6313	* the guest-CPU context.
6314	*
6315	* @returns VBox status code.
6316	* @param pVCpu Pointer to the VMCPU.
6317	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6318	* out-of-sync. Make sure to update the required fields
6319	* before using them.
6320	*
6321	* @remarks No-long-jump zone!!!
6322	*/
6323	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6324	{
6325	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6326	return VINF_SUCCESS;
6327
6328	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6329	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6330	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6331	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6332	{
6333	switch (pMsr->u32Msr)
6334	{
6335	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6336	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6337	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6338	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6339	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6340	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6341	break;
6342
6343	default:
6344	{
6345	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6346	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6347	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6348	}
6349	}
6350	}
6351
6352	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6353	return VINF_SUCCESS;
6354	}
6355
6356
6357	/**
6358	* Saves the guest control registers from the current VMCS into the guest-CPU
6359	* context.
6360	*
6361	* @returns VBox status code.
6362	* @param pVCpu Pointer to the VMCPU.
6363	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6364	* out-of-sync. Make sure to update the required fields
6365	* before using them.
6366	*
6367	* @remarks No-long-jump zone!!!
6368	*/
6369	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6370	{
6371	/* Guest CR0. Guest FPU. */
6372	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6373	AssertRCReturn(rc, rc);
6374
6375	/* Guest CR4. */
6376	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6377	AssertRCReturn(rc, rc);
6378
6379	/* Guest CR2 - updated always during the world-switch or in #PF. */
6380	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6381	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6382	{
6383	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6384	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6385
6386	PVM pVM = pVCpu->CTX_SUFF(pVM);
6387	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6388	\|\| ( pVM->hm.s.fNestedPaging
6389	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6390	{
6391	uint64_t u64Val = 0;
6392	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6393	if (pMixedCtx->cr3 != u64Val)
6394	{
6395	CPUMSetGuestCR3(pVCpu, u64Val);
6396	if (VMMRZCallRing3IsEnabled(pVCpu))
6397	{
6398	PGMUpdateCR3(pVCpu, u64Val);
6399	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6400	}
6401	else
6402	{
6403	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6404	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6405	}
6406	}
6407
6408	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6409	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6410	{
6411	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6412	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6413	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6414	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6415
6416	if (VMMRZCallRing3IsEnabled(pVCpu))
6417	{
6418	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6419	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6420	}
6421	else
6422	{
6423	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6424	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6425	}
6426	}
6427	}
6428
6429	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6430	}
6431
6432	/*
6433	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6434	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6435	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6436	*
6437	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6438	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6439	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6440	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6441	*
6442	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6443	*/
6444	if (VMMRZCallRing3IsEnabled(pVCpu))
6445	{
6446	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6447	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6448
6449	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6450	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6451
6452	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6453	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6454	}
6455
6456	return rc;
6457	}
6458
6459
6460	/**
6461	* Reads a guest segment register from the current VMCS into the guest-CPU
6462	* context.
6463	*
6464	* @returns VBox status code.
6465	* @param pVCpu Pointer to the VMCPU.
6466	* @param idxSel Index of the selector in the VMCS.
6467	* @param idxLimit Index of the segment limit in the VMCS.
6468	* @param idxBase Index of the segment base in the VMCS.
6469	* @param idxAccess Index of the access rights of the segment in the VMCS.
6470	* @param pSelReg Pointer to the segment selector.
6471	*
6472	* @remarks No-long-jump zone!!!
6473	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6474	* macro as that takes care of whether to read from the VMCS cache or
6475	* not.
6476	*/
6477	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6478	PCPUMSELREG pSelReg)
6479	{
6480	NOREF(pVCpu);
6481
6482	uint32_t u32Val = 0;
6483	int rc = VMXReadVmcs32(idxSel, &u32Val);
6484	AssertRCReturn(rc, rc);
6485	pSelReg->Sel = (uint16_t)u32Val;
6486	pSelReg->ValidSel = (uint16_t)u32Val;
6487	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6488
6489	rc = VMXReadVmcs32(idxLimit, &u32Val);
6490	AssertRCReturn(rc, rc);
6491	pSelReg->u32Limit = u32Val;
6492
6493	uint64_t u64Val = 0;
6494	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6495	AssertRCReturn(rc, rc);
6496	pSelReg->u64Base = u64Val;
6497
6498	rc = VMXReadVmcs32(idxAccess, &u32Val);
6499	AssertRCReturn(rc, rc);
6500	pSelReg->Attr.u = u32Val;
6501
6502	/*
6503	* If VT-x marks the segment as unusable, most other bits remain undefined:
6504	* - For CS the L, D and G bits have meaning.
6505	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6506	* - For the remaining data segments no bits are defined.
6507	*
6508	* The present bit and the unusable bit has been observed to be set at the
6509	* same time (the selector was supposed to be invalid as we started executing
6510	* a V8086 interrupt in ring-0).
6511	*
6512	* What should be important for the rest of the VBox code, is that the P bit is
6513	* cleared. Some of the other VBox code recognizes the unusable bit, but
6514	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6515	* safe side here, we'll strip off P and other bits we don't care about. If
6516	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6517	*
6518	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6519	*/
6520	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6521	{
6522	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6523
6524	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6525	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6526	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6527
6528	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6529	#ifdef DEBUG_bird
6530	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6531	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6532	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6533	#endif
6534	}
6535	return VINF_SUCCESS;
6536	}
6537
6538
6539	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6540	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6541	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6542	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6543	#else
6544	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6545	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6546	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6547	#endif
6548
6549
6550	/**
6551	* Saves the guest segment registers from the current VMCS into the guest-CPU
6552	* context.
6553	*
6554	* @returns VBox status code.
6555	* @param pVCpu Pointer to the VMCPU.
6556	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6557	* out-of-sync. Make sure to update the required fields
6558	* before using them.
6559	*
6560	* @remarks No-long-jump zone!!!
6561	*/
6562	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6563	{
6564	/* Guest segment registers. */
6565	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6566	{
6567	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6568	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6569	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6570	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6571	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6572	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6573	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6574
6575	/* Restore segment attributes for real-on-v86 mode hack. */
6576	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6577	{
6578	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6579	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6580	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6581	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6582	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6583	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6584	}
6585	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6586	}
6587
6588	return VINF_SUCCESS;
6589	}
6590
6591
6592	/**
6593	* Saves the guest descriptor table registers and task register from the current
6594	* VMCS into the guest-CPU context.
6595	*
6596	* @returns VBox status code.
6597	* @param pVCpu Pointer to the VMCPU.
6598	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6599	* out-of-sync. Make sure to update the required fields
6600	* before using them.
6601	*
6602	* @remarks No-long-jump zone!!!
6603	*/
6604	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6605	{
6606	int rc = VINF_SUCCESS;
6607
6608	/* Guest LDTR. */
6609	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6610	{
6611	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6612	AssertRCReturn(rc, rc);
6613	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6614	}
6615
6616	/* Guest GDTR. */
6617	uint64_t u64Val = 0;
6618	uint32_t u32Val = 0;
6619	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6620	{
6621	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6622	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6623	pMixedCtx->gdtr.pGdt = u64Val;
6624	pMixedCtx->gdtr.cbGdt = u32Val;
6625	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6626	}
6627
6628	/* Guest IDTR. */
6629	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6630	{
6631	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6632	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6633	pMixedCtx->idtr.pIdt = u64Val;
6634	pMixedCtx->idtr.cbIdt = u32Val;
6635	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6636	}
6637
6638	/* Guest TR. */
6639	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6640	{
6641	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6642	AssertRCReturn(rc, rc);
6643
6644	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6645	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6646	{
6647	rc = VMXLOCAL_READ_SEG(TR, tr);
6648	AssertRCReturn(rc, rc);
6649	}
6650	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6651	}
6652	return rc;
6653	}
6654
6655	#undef VMXLOCAL_READ_SEG
6656
6657
6658	/**
6659	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6660	* context.
6661	*
6662	* @returns VBox status code.
6663	* @param pVCpu Pointer to the VMCPU.
6664	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6665	* out-of-sync. Make sure to update the required fields
6666	* before using them.
6667	*
6668	* @remarks No-long-jump zone!!!
6669	*/
6670	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6671	{
6672	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6673	{
6674	if (!pVCpu->hm.s.fUsingHyperDR7)
6675	{
6676	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6677	uint32_t u32Val;
6678	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6679	pMixedCtx->dr[7] = u32Val;
6680	}
6681
6682	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6683	}
6684	return VINF_SUCCESS;
6685	}
6686
6687
6688	/**
6689	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6690	*
6691	* @returns VBox status code.
6692	* @param pVCpu Pointer to the VMCPU.
6693	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6694	* out-of-sync. Make sure to update the required fields
6695	* before using them.
6696	*
6697	* @remarks No-long-jump zone!!!
6698	*/
6699	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6700	{
6701	NOREF(pMixedCtx);
6702
6703	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6704	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6705	return VINF_SUCCESS;
6706	}
6707
6708
6709	/**
6710	* Saves the entire guest state from the currently active VMCS into the
6711	* guest-CPU context.
6712	*
6713	* This essentially VMREADs all guest-data.
6714	*
6715	* @returns VBox status code.
6716	* @param pVCpu Pointer to the VMCPU.
6717	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6718	* out-of-sync. Make sure to update the required fields
6719	* before using them.
6720	*/
6721	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6722	{
6723	Assert(pVCpu);
6724	Assert(pMixedCtx);
6725
6726	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6727	return VINF_SUCCESS;
6728
6729	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6730	again on the ring-3 callback path, there is no real need to. */
6731	if (VMMRZCallRing3IsEnabled(pVCpu))
6732	VMMR0LogFlushDisable(pVCpu);
6733	else
6734	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6735	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6736
6737	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6738	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6739
6740	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6741	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6742
6743	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6744	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6745
6746	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6747	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6748
6749	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6750	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6751
6752	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6753	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6754
6755	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6756	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6757
6758	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6759	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6760
6761	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6762	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6763
6764	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6765	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6766
6767	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6768	("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6769
6770	if (VMMRZCallRing3IsEnabled(pVCpu))
6771	VMMR0LogFlushEnable(pVCpu);
6772
6773	return VINF_SUCCESS;
6774	}
6775
6776
6777	/**
6778	* Saves basic guest registers needed for IEM instruction execution.
6779	*
6780	* @returns VBox status code (OR-able).
6781	* @param pVCpu Pointer to the cross context CPU data for the calling
6782	* EMT.
6783	* @param pMixedCtx Pointer to the CPU context of the guest.
6784	* @param fMemory Whether the instruction being executed operates on
6785	* memory or not. Only CR0 is synced up if clear.
6786	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6787	*/
6788	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6789	{
6790	/*
6791	* We assume all general purpose registers other than RSP are available.
6792	*
6793	* RIP is a must as it will be incremented or otherwise changed.
6794	*
6795	* RFLAGS are always required to figure the CPL.
6796	*
6797	* RSP isn't always required, however it's a GPR so frequently required.
6798	*
6799	* SS and CS are the only segment register needed if IEM doesn't do memory
6800	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6801	*
6802	* CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6803	* be required for memory accesses.
6804	*
6805	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
6806	*/
6807	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6808	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6809	if (fNeedRsp)
6810	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6811	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6812	if (!fMemory)
6813	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6814	else
6815	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6816	return rc;
6817	}
6818
6819
6820	/**
6821	* Ensures that we've got a complete basic context.
6822	*
6823	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
6824	* is for the interpreter.
6825	*
6826	* @returns VBox status code.
6827	* @param pVCpu Pointer to the VMCPU of the calling EMT.
6828	* @param pMixedCtx Pointer to the guest-CPU context which may have data
6829	* needing to be synced in.
6830	* @thread EMT(pVCpu)
6831	*/
6832	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6833	{
6834	/* Note! Since this is only applicable to VT-x, the implementation is placed
6835	in the VT-x part of the sources instead of the generic stuff. */
6836	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6837	return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6838	return VINF_SUCCESS;
6839	}
6840
6841
6842	/**
6843	* Check per-VM and per-VCPU force flag actions that require us to go back to
6844	* ring-3 for one reason or another.
6845	*
6846	* @returns VBox status code (information status code included).
6847	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6848	* ring-3.
6849	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6850	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6851	* interrupts)
6852	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6853	* all EMTs to be in ring-3.
6854	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6855	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6856	* to the EM loop.
6857	*
6858	* @param pVM Pointer to the VM.
6859	* @param pVCpu Pointer to the VMCPU.
6860	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6861	* out-of-sync. Make sure to update the required fields
6862	* before using them.
6863	*/
6864	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6865	{
6866	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6867
6868	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6869	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6870	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6871	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6872	{
6873	/* We need the control registers now, make sure the guest-CPU context is updated. */
6874	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6875	AssertRCReturn(rc3, rc3);
6876
6877	/* Pending HM CR3 sync. */
6878	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6879	{
6880	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6881	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6882	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6883	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6884	}
6885
6886	/* Pending HM PAE PDPEs. */
6887	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6888	{
6889	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6890	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6891	}
6892
6893	/* Pending PGM C3 sync. */
6894	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6895	{
6896	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6897	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6898	if (rc2 != VINF_SUCCESS)
6899	{
6900	AssertRC(rc2);
6901	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6902	return rc2;
6903	}
6904	}
6905
6906	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6907	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6908	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6909	{
6910	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6911	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6912	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6913	return rc2;
6914	}
6915
6916	/* Pending VM request packets, such as hardware interrupts. */
6917	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6918	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6919	{
6920	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6921	return VINF_EM_PENDING_REQUEST;
6922	}
6923
6924	/* Pending PGM pool flushes. */
6925	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6926	{
6927	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6928	return VINF_PGM_POOL_FLUSH_PENDING;
6929	}
6930
6931	/* Pending DMA requests. */
6932	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6933	{
6934	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6935	return VINF_EM_RAW_TO_R3;
6936	}
6937	}
6938
6939	return VINF_SUCCESS;
6940	}
6941
6942
6943	/**
6944	* Converts any TRPM trap into a pending HM event. This is typically used when
6945	* entering from ring-3 (not longjmp returns).
6946	*
6947	* @param pVCpu Pointer to the VMCPU.
6948	*/
6949	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6950	{
6951	Assert(TRPMHasTrap(pVCpu));
6952	Assert(!pVCpu->hm.s.Event.fPending);
6953
6954	uint8_t uVector;
6955	TRPMEVENT enmTrpmEvent;
6956	RTGCUINT uErrCode;
6957	RTGCUINTPTR GCPtrFaultAddress;
6958	uint8_t cbInstr;
6959
6960	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6961	AssertRC(rc);
6962
6963	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6964	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6965	if (enmTrpmEvent == TRPM_TRAP)
6966	{
6967	switch (uVector)
6968	{
6969	case X86_XCPT_NMI:
6970	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6971	break;
6972
6973	case X86_XCPT_BP:
6974	case X86_XCPT_OF:
6975	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6976	break;
6977
6978	case X86_XCPT_PF:
6979	case X86_XCPT_DF:
6980	case X86_XCPT_TS:
6981	case X86_XCPT_NP:
6982	case X86_XCPT_SS:
6983	case X86_XCPT_GP:
6984	case X86_XCPT_AC:
6985	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6986	/* no break! */
6987	default:
6988	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6989	break;
6990	}
6991	}
6992	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6993	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6994	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6995	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6996	else
6997	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6998
6999	rc = TRPMResetTrap(pVCpu);
7000	AssertRC(rc);
7001	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7002	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7003
7004	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7005	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
7006	}
7007
7008
7009	/**
7010	* Converts any pending HM event into a TRPM trap. Typically used when leaving
7011	* VT-x to execute any instruction.
7012	*
7013	* @param pvCpu Pointer to the VMCPU.
7014	*/
7015	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7016	{
7017	Assert(pVCpu->hm.s.Event.fPending);
7018
7019	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7020	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7021	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7022	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7023
7024	/* If a trap was already pending, we did something wrong! */
7025	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7026
7027	TRPMEVENT enmTrapType;
7028	switch (uVectorType)
7029	{
7030	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7031	enmTrapType = TRPM_HARDWARE_INT;
7032	break;
7033
7034	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7035	enmTrapType = TRPM_SOFTWARE_INT;
7036	break;
7037
7038	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7039	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7040	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7041	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7042	enmTrapType = TRPM_TRAP;
7043	break;
7044
7045	default:
7046	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7047	enmTrapType = TRPM_32BIT_HACK;
7048	break;
7049	}
7050
7051	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7052
7053	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7054	AssertRC(rc);
7055
7056	if (fErrorCodeValid)
7057	TRPMSetErrorCode(pVCpu, uErrorCode);
7058
7059	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7060	&& uVector == X86_XCPT_PF)
7061	{
7062	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7063	}
7064	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7065	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7066	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7067	{
7068	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7069	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7070	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7071	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7072	}
7073	pVCpu->hm.s.Event.fPending = false;
7074	}
7075
7076
7077	/**
7078	* Does the necessary state syncing before returning to ring-3 for any reason
7079	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7080	*
7081	* @returns VBox status code.
7082	* @param pVM Pointer to the VM.
7083	* @param pVCpu Pointer to the VMCPU.
7084	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7085	* be out-of-sync. Make sure to update the required
7086	* fields before using them.
7087	* @param fSaveGuestState Whether to save the guest state or not.
7088	*
7089	* @remarks No-long-jmp zone!!!
7090	*/
7091	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7092	{
7093	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7094	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7095
7096	RTCPUID idCpu = RTMpCpuId();
7097	Log4Func(("HostCpuId=%u\n", idCpu));
7098
7099	/*
7100	* !!! IMPORTANT !!!
7101	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7102	*/
7103
7104	/* Save the guest state if necessary. */
7105	if ( fSaveGuestState
7106	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7107	{
7108	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7109	AssertRCReturn(rc, rc);
7110	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7111	}
7112
7113	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7114	if (CPUMIsGuestFPUStateActive(pVCpu))
7115	{
7116	/* We shouldn't reload CR0 without saving it first. */
7117	if (!fSaveGuestState)
7118	{
7119	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7120	AssertRCReturn(rc, rc);
7121	}
7122	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7123	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7124	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7125	}
7126
7127	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7128	#ifdef VBOX_STRICT
7129	if (CPUMIsHyperDebugStateActive(pVCpu))
7130	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7131	#endif
7132	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7133	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7134	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7135	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7136
7137	#if HC_ARCH_BITS == 64
7138	/* Restore host-state bits that VT-x only restores partially. */
7139	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7140	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7141	{
7142	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7143	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7144	}
7145	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7146	#endif
7147
7148	#if HC_ARCH_BITS == 64
7149	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7150	if ( pVM->hm.s.fAllow64BitGuests
7151	&& pVCpu->hm.s.vmx.fLazyMsrs)
7152	{
7153	/* We shouldn't reload the guest MSRs without saving it first. */
7154	if (!fSaveGuestState)
7155	{
7156	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7157	AssertRCReturn(rc, rc);
7158	}
7159	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7160	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7161	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7162	}
7163	#endif
7164
7165	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7166	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7167
7168	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7169	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7170	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7171	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7172	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7173	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7174	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7175	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7176
7177	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7178
7179	/** @todo This partially defeats the purpose of having preemption hooks.
7180	* The problem is, deregistering the hooks should be moved to a place that
7181	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7182	* context.
7183	*/
7184	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7185	{
7186	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7187	AssertRCReturn(rc, rc);
7188
7189	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7190	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7191	}
7192	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7193	NOREF(idCpu);
7194
7195	return VINF_SUCCESS;
7196	}
7197
7198
7199	/**
7200	* Leaves the VT-x session.
7201	*
7202	* @returns VBox status code.
7203	* @param pVM Pointer to the VM.
7204	* @param pVCpu Pointer to the VMCPU.
7205	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7206	* out-of-sync. Make sure to update the required fields
7207	* before using them.
7208	*
7209	* @remarks No-long-jmp zone!!!
7210	*/
7211	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7212	{
7213	HM_DISABLE_PREEMPT();
7214	HMVMX_ASSERT_CPU_SAFE();
7215	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7216	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7217
7218	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7219	and done this from the VMXR0ThreadCtxCallback(). */
7220	if (!pVCpu->hm.s.fLeaveDone)
7221	{
7222	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7223	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7224	pVCpu->hm.s.fLeaveDone = true;
7225	}
7226	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7227
7228	/*
7229	* !!! IMPORTANT !!!
7230	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7231	*/
7232
7233	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7234	/** @todo Deregistering here means we need to VMCLEAR always
7235	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7236	VMMR0ThreadCtxHooksDeregister(pVCpu);
7237
7238	/* Leave HM context. This takes care of local init (term). */
7239	int rc = HMR0LeaveCpu(pVCpu);
7240
7241	HM_RESTORE_PREEMPT();
7242	return rc;
7243	}
7244
7245
7246	/**
7247	* Does the necessary state syncing before doing a longjmp to ring-3.
7248	*
7249	* @returns VBox status code.
7250	* @param pVM Pointer to the VM.
7251	* @param pVCpu Pointer to the VMCPU.
7252	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7253	* out-of-sync. Make sure to update the required fields
7254	* before using them.
7255	*
7256	* @remarks No-long-jmp zone!!!
7257	*/
7258	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7259	{
7260	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7261	}
7262
7263
7264	/**
7265	* Take necessary actions before going back to ring-3.
7266	*
7267	* An action requires us to go back to ring-3. This function does the necessary
7268	* steps before we can safely return to ring-3. This is not the same as longjmps
7269	* to ring-3, this is voluntary and prepares the guest so it may continue
7270	* executing outside HM (recompiler/IEM).
7271	*
7272	* @returns VBox status code.
7273	* @param pVM Pointer to the VM.
7274	* @param pVCpu Pointer to the VMCPU.
7275	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7276	* out-of-sync. Make sure to update the required fields
7277	* before using them.
7278	* @param rcExit The reason for exiting to ring-3. Can be
7279	* VINF_VMM_UNKNOWN_RING3_CALL.
7280	*/
7281	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7282	{
7283	Assert(pVM);
7284	Assert(pVCpu);
7285	Assert(pMixedCtx);
7286	HMVMX_ASSERT_PREEMPT_SAFE();
7287
7288	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7289	{
7290	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7291	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7292	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7293	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7294	}
7295
7296	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7297	VMMRZCallRing3Disable(pVCpu);
7298	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7299
7300	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7301	if (pVCpu->hm.s.Event.fPending)
7302	{
7303	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7304	Assert(!pVCpu->hm.s.Event.fPending);
7305	}
7306
7307	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7308	and if we're injecting an event we should have a TRPM trap pending. */
7309	Assert(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu));
7310	Assert(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu));
7311
7312	/* Save guest state and restore host state bits. */
7313	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7314	AssertRCReturn(rc, rc);
7315	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7316	/* Thread-context hooks are unregistered at this point!!! */
7317
7318	/* Sync recompiler state. */
7319	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7320	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7321	\| CPUM_CHANGED_LDTR
7322	\| CPUM_CHANGED_GDTR
7323	\| CPUM_CHANGED_IDTR
7324	\| CPUM_CHANGED_TR
7325	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7326	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7327	if ( pVM->hm.s.fNestedPaging
7328	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7329	{
7330	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7331	}
7332
7333	Assert(!pVCpu->hm.s.fClearTrapFlag);
7334
7335	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7336	if (rcExit != VINF_EM_RAW_INTERRUPT)
7337	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7338
7339	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7340
7341	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7342	VMMRZCallRing3RemoveNotification(pVCpu);
7343	VMMRZCallRing3Enable(pVCpu);
7344
7345	return rc;
7346	}
7347
7348
7349	/**
7350	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7351	* longjump to ring-3 and possibly get preempted.
7352	*
7353	* @returns VBox status code.
7354	* @param pVCpu Pointer to the VMCPU.
7355	* @param enmOperation The operation causing the ring-3 longjump.
7356	* @param pvUser Opaque pointer to the guest-CPU context. The data
7357	* may be out-of-sync. Make sure to update the required
7358	* fields before using them.
7359	*/
7360	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7361	{
7362	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7363	{
7364	/*
7365	* !!! IMPORTANT !!!
7366	* If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7367	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7368	*/
7369	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER; \
7370	VMMRZCallRing3RemoveNotification(pVCpu);
7371	VMMRZCallRing3Disable(pVCpu);
7372	RTThreadPreemptDisable(&PreemptState);
7373
7374	PVM pVM = pVCpu->CTX_SUFF(pVM);
7375	if (CPUMIsGuestFPUStateActive(pVCpu))
7376	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7377
7378	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7379
7380	#if HC_ARCH_BITS == 64
7381	/* Restore host-state bits that VT-x only restores partially. */
7382	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7383	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7384	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7385	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7386
7387	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7388	if ( pVM->hm.s.fAllow64BitGuests
7389	&& pVCpu->hm.s.vmx.fLazyMsrs)
7390	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7391	#endif
7392	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7393	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7394	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7395	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7396	{
7397	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7398	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7399	}
7400
7401	VMMR0ThreadCtxHooksDeregister(pVCpu);
7402	HMR0LeaveCpu(pVCpu);
7403	RTThreadPreemptRestore(&PreemptState);
7404	return VINF_SUCCESS;
7405	}
7406
7407	Assert(pVCpu);
7408	Assert(pvUser);
7409	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7410	HMVMX_ASSERT_PREEMPT_SAFE();
7411
7412	VMMRZCallRing3Disable(pVCpu);
7413	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7414
7415	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7416	enmOperation));
7417
7418	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7419	AssertRCReturn(rc, rc);
7420
7421	VMMRZCallRing3Enable(pVCpu);
7422	return VINF_SUCCESS;
7423	}
7424
7425
7426	/**
7427	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7428	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7429	*
7430	* @param pVCpu Pointer to the VMCPU.
7431	*/
7432	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7433	{
7434	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7435	{
7436	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7437	{
7438	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7439	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7440	AssertRC(rc);
7441	Log4(("Setup interrupt-window exiting\n"));
7442	}
7443	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7444	}
7445
7446
7447	/**
7448	* Clears the interrupt-window exiting control in the VMCS.
7449	*
7450	* @param pVCpu Pointer to the VMCPU.
7451	*/
7452	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7453	{
7454	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7455	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7456	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7457	AssertRC(rc);
7458	Log4(("Cleared interrupt-window exiting\n"));
7459	}
7460
7461
7462	/**
7463	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7464	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7465	*
7466	* @param pVCpu Pointer to the VMCPU.
7467	*/
7468	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7469	{
7470	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7471	{
7472	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7473	{
7474	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7475	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7476	AssertRC(rc);
7477	Log4(("Setup NMI-window exiting\n"));
7478	}
7479	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7480	}
7481
7482
7483	/**
7484	* Clears the NMI-window exiting control in the VMCS.
7485	*
7486	* @param pVCpu Pointer to the VMCPU.
7487	*/
7488	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7489	{
7490	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7491	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7492	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7493	AssertRC(rc);
7494	Log4(("Cleared NMI-window exiting\n"));
7495	}
7496
7497
7498	/**
7499	* Evaluates the event to be delivered to the guest and sets it as the pending
7500	* event.
7501	*
7502	* @param pVCpu Pointer to the VMCPU.
7503	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7504	* out-of-sync. Make sure to update the required fields
7505	* before using them.
7506	*/
7507	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7508	{
7509	Assert(!pVCpu->hm.s.Event.fPending);
7510
7511	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7512	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7513	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7514	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7515	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7516
7517	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7518	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7519	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7520	Assert(!TRPMHasTrap(pVCpu));
7521
7522	/*
7523	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7524	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7525	*/
7526	/** @todo SMI. SMIs take priority over NMIs. */
7527	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7528	{
7529	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7530	if ( !fBlockNmi
7531	&& !fBlockSti
7532	&& !fBlockMovSS)
7533	{
7534	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7535	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7536	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7537
7538	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7539	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7540	}
7541	else
7542	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7543	}
7544	/*
7545	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7546	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7547	*/
7548	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7549	&& !pVCpu->hm.s.fSingleInstruction)
7550	{
7551	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7552	AssertRC(rc);
7553	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7554	if ( !fBlockInt
7555	&& !fBlockSti
7556	&& !fBlockMovSS)
7557	{
7558	uint8_t u8Interrupt;
7559	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7560	if (RT_SUCCESS(rc))
7561	{
7562	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7563	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7564	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7565
7566	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7567	}
7568	else
7569	{
7570	/** @todo Does this actually happen? If not turn it into an assertion. */
7571	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7572	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7573	}
7574	}
7575	else
7576	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7577	}
7578	}
7579
7580
7581	/**
7582	* Sets a pending-debug exception to be delivered to the guest if the guest is
7583	* single-stepping.
7584	*
7585	* @param pVCpu Pointer to the VMCPU.
7586	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7587	* out-of-sync. Make sure to update the required fields
7588	* before using them.
7589	*/
7590	DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7591	{
7592	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7593	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7594	{
7595	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7596	AssertRC(rc);
7597	}
7598	}
7599
7600
7601	/**
7602	* Injects any pending events into the guest if the guest is in a state to
7603	* receive them.
7604	*
7605	* @returns VBox status code (informational status codes included).
7606	* @param pVCpu Pointer to the VMCPU.
7607	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7608	* out-of-sync. Make sure to update the required fields
7609	* before using them.
7610	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7611	* return VINF_EM_DBG_STEPPED if the event was
7612	* dispatched directly.
7613	*/
7614	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7615	{
7616	HMVMX_ASSERT_PREEMPT_SAFE();
7617	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7618
7619	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7620	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7621	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7622	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7623
7624	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7625	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7626	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7627	Assert(!TRPMHasTrap(pVCpu));
7628
7629	int rc = VINF_SUCCESS;
7630	if (pVCpu->hm.s.Event.fPending)
7631	{
7632	/*
7633	* Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7634	* VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7635	* ended up enabling interrupts outside VT-x.
7636	*/
7637	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7638	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7639	&& uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7640	{
7641	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7642	hmR0VmxClearIntWindowExitVmcs(pVCpu);
7643	}
7644
7645	#ifdef VBOX_STRICT
7646	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7647	{
7648	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7649	Assert(!fBlockInt);
7650	Assert(!fBlockSti);
7651	Assert(!fBlockMovSS);
7652	}
7653	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7654	{
7655	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7656	Assert(!fBlockSti);
7657	Assert(!fBlockMovSS);
7658	Assert(!fBlockNmi);
7659	}
7660	#endif
7661	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7662	(uint8_t)uIntType));
7663	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7664	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping, &uIntrState);
7665	AssertRCReturn(rc, rc);
7666
7667	/* Update the interruptibility-state as it could have been changed by
7668	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7669	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7670	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7671
7672	#ifdef VBOX_WITH_STATISTICS
7673	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7674	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7675	else
7676	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7677	#endif
7678	}
7679
7680	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7681	if ( fBlockSti
7682	\|\| fBlockMovSS)
7683	{
7684	if ( !pVCpu->hm.s.fSingleInstruction
7685	&& !DBGFIsStepping(pVCpu))
7686	{
7687	/*
7688	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7689	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7690	* See Intel spec. 27.3.4 "Saving Non-Register State".
7691	*/
7692	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7693	AssertRCReturn(rc2, rc2);
7694	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7695	}
7696	else if (pMixedCtx->eflags.Bits.u1TF)
7697	{
7698	/*
7699	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7700	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7701	*/
7702	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7703	uIntrState = 0;
7704	}
7705	}
7706
7707	/*
7708	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7709	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7710	*/
7711	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7712	AssertRC(rc2);
7713
7714	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET \|\| (rc == VINF_EM_DBG_STEPPED && fStepping));
7715	NOREF(fBlockMovSS); NOREF(fBlockSti);
7716	return rc;
7717	}
7718
7719
7720	/**
7721	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7722	*
7723	* @param pVCpu Pointer to the VMCPU.
7724	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7725	* out-of-sync. Make sure to update the required fields
7726	* before using them.
7727	*/
7728	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7729	{
7730	NOREF(pMixedCtx);
7731	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7732	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7733	}
7734
7735
7736	/**
7737	* Injects a double-fault (#DF) exception into the VM.
7738	*
7739	* @returns VBox status code (informational status code included).
7740	* @param pVCpu Pointer to the VMCPU.
7741	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7742	* out-of-sync. Make sure to update the required fields
7743	* before using them.
7744	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7745	* and should return VINF_EM_DBG_STEPPED if the event
7746	* is injected directly (register modified by us, not
7747	* by hardware on VM-entry).
7748	* @param puIntrState Pointer to the current guest interruptibility-state.
7749	* This interruptibility-state will be updated if
7750	* necessary. This cannot not be NULL.
7751	*/
7752	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7753	{
7754	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7755	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7756	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7757	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7758	fStepping, puIntrState);
7759	}
7760
7761
7762	/**
7763	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7764	*
7765	* @param pVCpu Pointer to the VMCPU.
7766	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7767	* out-of-sync. Make sure to update the required fields
7768	* before using them.
7769	*/
7770	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7771	{
7772	NOREF(pMixedCtx);
7773	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7774	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7775	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7776	}
7777
7778
7779	/**
7780	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7781	*
7782	* @param pVCpu Pointer to the VMCPU.
7783	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7784	* out-of-sync. Make sure to update the required fields
7785	* before using them.
7786	* @param cbInstr The value of RIP that is to be pushed on the guest
7787	* stack.
7788	*/
7789	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7790	{
7791	NOREF(pMixedCtx);
7792	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7793	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7794	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7795	}
7796
7797
7798	/**
7799	* Injects a general-protection (#GP) fault into the VM.
7800	*
7801	* @returns VBox status code (informational status code included).
7802	* @param pVCpu Pointer to the VMCPU.
7803	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7804	* out-of-sync. Make sure to update the required fields
7805	* before using them.
7806	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7807	* mode, i.e. in real-mode it's not valid).
7808	* @param u32ErrorCode The error code associated with the #GP.
7809	* @param fStepping Whether we're running in
7810	* hmR0VmxRunGuestCodeStep() and should return
7811	* VINF_EM_DBG_STEPPED if the event is injected
7812	* directly (register modified by us, not by
7813	* hardware on VM-entry).
7814	* @param puIntrState Pointer to the current guest interruptibility-state.
7815	* This interruptibility-state will be updated if
7816	* necessary. This cannot not be NULL.
7817	*/
7818	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7819	bool fStepping, uint32_t *puIntrState)
7820	{
7821	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7822	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7823	if (fErrorCodeValid)
7824	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7825	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7826	fStepping, puIntrState);
7827	}
7828
7829
7830	/**
7831	* Sets a general-protection (#GP) exception as pending-for-injection into the
7832	* VM.
7833	*
7834	* @param pVCpu Pointer to the VMCPU.
7835	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7836	* out-of-sync. Make sure to update the required fields
7837	* before using them.
7838	* @param u32ErrorCode The error code associated with the #GP.
7839	*/
7840	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7841	{
7842	NOREF(pMixedCtx);
7843	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7844	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7845	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7846	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
7847	}
7848
7849
7850	/**
7851	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7852	*
7853	* @param pVCpu Pointer to the VMCPU.
7854	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7855	* out-of-sync. Make sure to update the required fields
7856	* before using them.
7857	* @param uVector The software interrupt vector number.
7858	* @param cbInstr The value of RIP that is to be pushed on the guest
7859	* stack.
7860	*/
7861	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7862	{
7863	NOREF(pMixedCtx);
7864	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7865	if ( uVector == X86_XCPT_BP
7866	\|\| uVector == X86_XCPT_OF)
7867	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7868	else
7869	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7870	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7871	}
7872
7873
7874	/**
7875	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7876	* stack.
7877	*
7878	* @returns VBox status code (information status code included).
7879	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7880	* @param pVM Pointer to the VM.
7881	* @param pMixedCtx Pointer to the guest-CPU context.
7882	* @param uValue The value to push to the guest stack.
7883	*/
7884	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7885	{
7886	/*
7887	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7888	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7889	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7890	*/
7891	if (pMixedCtx->sp == 1)
7892	return VINF_EM_RESET;
7893	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7894	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7895	AssertRCReturn(rc, rc);
7896	return rc;
7897	}
7898
7899
7900	/**
7901	* Injects an event into the guest upon VM-entry by updating the relevant fields
7902	* in the VM-entry area in the VMCS.
7903	*
7904	* @returns VBox status code (informational error codes included).
7905	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7906	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7907	*
7908	* @param pVCpu Pointer to the VMCPU.
7909	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7910	* be out-of-sync. Make sure to update the required
7911	* fields before using them.
7912	* @param u64IntInfo The VM-entry interruption-information field.
7913	* @param cbInstr The VM-entry instruction length in bytes (for
7914	* software interrupts, exceptions and privileged
7915	* software exceptions).
7916	* @param u32ErrCode The VM-entry exception error code.
7917	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7918	* @param puIntrState Pointer to the current guest interruptibility-state.
7919	* This interruptibility-state will be updated if
7920	* necessary. This cannot not be NULL.
7921	* @param fStepping Whether we're running in
7922	* hmR0VmxRunGuestCodeStep() and should return
7923	* VINF_EM_DBG_STEPPED if the event is injected
7924	* directly (register modified by us, not by
7925	* hardware on VM-entry).
7926	*
7927	* @remarks Requires CR0!
7928	* @remarks No-long-jump zone!!!
7929	*/
7930	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7931	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *puIntrState)
7932	{
7933	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7934	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7935	Assert(puIntrState);
7936	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7937
7938	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7939	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7940
7941	#ifdef VBOX_STRICT
7942	/* Validate the error-code-valid bit for hardware exceptions. */
7943	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7944	{
7945	switch (uVector)
7946	{
7947	case X86_XCPT_PF:
7948	case X86_XCPT_DF:
7949	case X86_XCPT_TS:
7950	case X86_XCPT_NP:
7951	case X86_XCPT_SS:
7952	case X86_XCPT_GP:
7953	case X86_XCPT_AC:
7954	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7955	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7956	/* fallthru */
7957	default:
7958	break;
7959	}
7960	}
7961	#endif
7962
7963	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7964	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7965	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7966
7967	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7968
7969	/* We require CR0 to check if the guest is in real-mode. */
7970	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7971	AssertRCReturn(rc, rc);
7972
7973	/*
7974	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7975	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7976	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7977	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7978	*/
7979	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7980	{
7981	PVM pVM = pVCpu->CTX_SUFF(pVM);
7982	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7983	{
7984	Assert(PDMVmmDevHeapIsEnabled(pVM));
7985	Assert(pVM->hm.s.vmx.pRealModeTSS);
7986
7987	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7988	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7989	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7990	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7991	AssertRCReturn(rc, rc);
7992	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7993
7994	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7995	size_t const cbIdtEntry = sizeof(X86IDTR16);
7996	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7997	{
7998	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7999	if (uVector == X86_XCPT_DF)
8000	return VINF_EM_RESET;
8001
8002	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8003	if (uVector == X86_XCPT_GP)
8004	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8005
8006	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8007	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8008	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
8009	fStepping, puIntrState);
8010	}
8011
8012	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8013	uint16_t uGuestIp = pMixedCtx->ip;
8014	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8015	{
8016	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8017	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8018	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8019	}
8020	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8021	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8022
8023	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8024	X86IDTR16 IdtEntry;
8025	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8026	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8027	AssertRCReturn(rc, rc);
8028
8029	/* Construct the stack frame for the interrupt/exception handler. */
8030	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8031	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8032	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8033	AssertRCReturn(rc, rc);
8034
8035	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8036	if (rc == VINF_SUCCESS)
8037	{
8038	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8039	pMixedCtx->rip = IdtEntry.offSel;
8040	pMixedCtx->cs.Sel = IdtEntry.uSel;
8041	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8042	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8043	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8044	&& uVector == X86_XCPT_PF)
8045	pMixedCtx->cr2 = GCPtrFaultAddress;
8046
8047	/* If any other guest-state bits are changed here, make sure to update
8048	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8049	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8050	\| HM_CHANGED_GUEST_RIP
8051	\| HM_CHANGED_GUEST_RFLAGS
8052	\| HM_CHANGED_GUEST_RSP);
8053
8054	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8055	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8056	{
8057	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8058	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8059	Log4(("Clearing inhibition due to STI.\n"));
8060	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8061	}
8062	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8063	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8064
8065	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8066	it, if we are returning to ring-3 before executing guest code. */
8067	pVCpu->hm.s.Event.fPending = false;
8068
8069	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8070	if (fStepping)
8071	rc = VINF_EM_DBG_STEPPED;
8072	}
8073	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET \|\| (rc == VINF_EM_DBG_STEPPED && fStepping));
8074	return rc;
8075	}
8076
8077	/*
8078	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8079	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8080	*/
8081	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8082	}
8083
8084	/* Validate. */
8085	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8086	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8087	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8088
8089	/* Inject. */
8090	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8091	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8092	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8093	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8094
8095	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8096	&& uVector == X86_XCPT_PF)
8097	pMixedCtx->cr2 = GCPtrFaultAddress;
8098
8099	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8100	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8101
8102	AssertRCReturn(rc, rc);
8103	return rc;
8104	}
8105
8106
8107	/**
8108	* Clears the interrupt-window exiting control in the VMCS and if necessary
8109	* clears the current event in the VMCS as well.
8110	*
8111	* @returns VBox status code.
8112	* @param pVCpu Pointer to the VMCPU.
8113	*
8114	* @remarks Use this function only to clear events that have not yet been
8115	* delivered to the guest but are injected in the VMCS!
8116	* @remarks No-long-jump zone!!!
8117	*/
8118	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
8119	{
8120	int rc;
8121	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8122
8123	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8124	{
8125	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8126	Assert(!pVCpu->hm.s.Event.fPending);
8127	}
8128
8129	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8130	{
8131	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8132	Assert(!pVCpu->hm.s.Event.fPending);
8133	}
8134
8135	if (!pVCpu->hm.s.Event.fPending)
8136	return;
8137
8138	#ifdef VBOX_STRICT
8139	uint32_t u32EntryInfo;
8140	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8141	AssertRC(rc);
8142	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8143	#endif
8144
8145	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8146	AssertRC(rc);
8147
8148	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8149	AssertRC(rc);
8150
8151	/* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8152	care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8153	}
8154
8155
8156	/**
8157	* Enters the VT-x session.
8158	*
8159	* @returns VBox status code.
8160	* @param pVM Pointer to the VM.
8161	* @param pVCpu Pointer to the VMCPU.
8162	* @param pCpu Pointer to the CPU info struct.
8163	*/
8164	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8165	{
8166	AssertPtr(pVM);
8167	AssertPtr(pVCpu);
8168	Assert(pVM->hm.s.vmx.fSupported);
8169	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8170	NOREF(pCpu); NOREF(pVM);
8171
8172	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8173	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8174
8175	#ifdef VBOX_STRICT
8176	/* Make sure we're in VMX root mode. */
8177	RTCCUINTREG u32HostCR4 = ASMGetCR4();
8178	if (!(u32HostCR4 & X86_CR4_VMXE))
8179	{
8180	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8181	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8182	}
8183	#endif
8184
8185	/*
8186	* Load the VCPU's VMCS as the current (and active) one.
8187	*/
8188	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8189	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8190	if (RT_FAILURE(rc))
8191	return rc;
8192
8193	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8194	pVCpu->hm.s.fLeaveDone = false;
8195	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8196
8197	return VINF_SUCCESS;
8198	}
8199
8200
8201	/**
8202	* The thread-context callback (only on platforms which support it).
8203	*
8204	* @param enmEvent The thread-context event.
8205	* @param pVCpu Pointer to the VMCPU.
8206	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8207	* @thread EMT(pVCpu)
8208	*/
8209	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8210	{
8211	NOREF(fGlobalInit);
8212
8213	switch (enmEvent)
8214	{
8215	case RTTHREADCTXEVENT_PREEMPTING:
8216	{
8217	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8218	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8219	VMCPU_ASSERT_EMT(pVCpu);
8220
8221	PVM pVM = pVCpu->CTX_SUFF(pVM);
8222	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8223
8224	/* No longjmps (logger flushes, locks) in this fragile context. */
8225	VMMRZCallRing3Disable(pVCpu);
8226	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8227
8228	/*
8229	* Restore host-state (FPU, debug etc.)
8230	*/
8231	if (!pVCpu->hm.s.fLeaveDone)
8232	{
8233	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8234	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8235	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8236	pVCpu->hm.s.fLeaveDone = true;
8237	}
8238
8239	/* Leave HM context, takes care of local init (term). */
8240	int rc = HMR0LeaveCpu(pVCpu);
8241	AssertRC(rc); NOREF(rc);
8242
8243	/* Restore longjmp state. */
8244	VMMRZCallRing3Enable(pVCpu);
8245	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8246	break;
8247	}
8248
8249	case RTTHREADCTXEVENT_RESUMED:
8250	{
8251	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8252	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8253	VMCPU_ASSERT_EMT(pVCpu);
8254
8255	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8256	VMMRZCallRing3Disable(pVCpu);
8257	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8258
8259	/* Initialize the bare minimum state required for HM. This takes care of
8260	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8261	int rc = HMR0EnterCpu(pVCpu);
8262	AssertRC(rc);
8263	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8264
8265	/* Load the active VMCS as the current one. */
8266	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8267	{
8268	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8269	AssertRC(rc); NOREF(rc);
8270	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8271	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8272	}
8273	pVCpu->hm.s.fLeaveDone = false;
8274
8275	/* Restore longjmp state. */
8276	VMMRZCallRing3Enable(pVCpu);
8277	break;
8278	}
8279
8280	default:
8281	break;
8282	}
8283	}
8284
8285
8286	/**
8287	* Saves the host state in the VMCS host-state.
8288	* Sets up the VM-exit MSR-load area.
8289	*
8290	* The CPU state will be loaded from these fields on every successful VM-exit.
8291	*
8292	* @returns VBox status code.
8293	* @param pVM Pointer to the VM.
8294	* @param pVCpu Pointer to the VMCPU.
8295	*
8296	* @remarks No-long-jump zone!!!
8297	*/
8298	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8299	{
8300	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8301
8302	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8303	return VINF_SUCCESS;
8304
8305	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8306	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8307
8308	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8309	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8310
8311	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8312	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8313
8314	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8315	return rc;
8316	}
8317
8318
8319	/**
8320	* Saves the host state in the VMCS host-state.
8321	*
8322	* @returns VBox status code.
8323	* @param pVM Pointer to the VM.
8324	* @param pVCpu Pointer to the VMCPU.
8325	*
8326	* @remarks No-long-jump zone!!!
8327	*/
8328	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8329	{
8330	AssertPtr(pVM);
8331	AssertPtr(pVCpu);
8332
8333	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8334
8335	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8336	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8337	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8338	return hmR0VmxSaveHostState(pVM, pVCpu);
8339	}
8340
8341
8342	/**
8343	* Loads the guest state into the VMCS guest-state area.
8344	*
8345	* The will typically be done before VM-entry when the guest-CPU state and the
8346	* VMCS state may potentially be out of sync.
8347	*
8348	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8349	* VM-entry controls.
8350	* Sets up the appropriate VMX non-root function to execute guest code based on
8351	* the guest CPU mode.
8352	*
8353	* @returns VBox status code.
8354	* @param pVM Pointer to the VM.
8355	* @param pVCpu Pointer to the VMCPU.
8356	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8357	* out-of-sync. Make sure to update the required fields
8358	* before using them.
8359	*
8360	* @remarks No-long-jump zone!!!
8361	*/
8362	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8363	{
8364	AssertPtr(pVM);
8365	AssertPtr(pVCpu);
8366	AssertPtr(pMixedCtx);
8367	HMVMX_ASSERT_PREEMPT_SAFE();
8368
8369	VMMRZCallRing3Disable(pVCpu);
8370	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8371
8372	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8373
8374	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8375
8376	/* Determine real-on-v86 mode. */
8377	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8378	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8379	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8380	{
8381	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8382	}
8383
8384	/*
8385	* Load the guest-state into the VMCS.
8386	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8387	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8388	*/
8389	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8390	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8391
8392	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8393	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8394	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8395
8396	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8397	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8398	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8399
8400	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8401	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8402
8403	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8404	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8405
8406	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8407	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8408	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8409
8410	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8411	determine we don't have to swap EFER after all. */
8412	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8413	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8414
8415	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8416	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8417
8418	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8419	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8420
8421	/*
8422	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8423	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8424	*/
8425	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8426	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8427
8428	/* Clear any unused and reserved bits. */
8429	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8430
8431	VMMRZCallRing3Enable(pVCpu);
8432
8433	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8434	return rc;
8435	}
8436
8437
8438	/**
8439	* Loads the state shared between the host and guest into the VMCS.
8440	*
8441	* @param pVM Pointer to the VM.
8442	* @param pVCpu Pointer to the VMCPU.
8443	* @param pCtx Pointer to the guest-CPU context.
8444	*
8445	* @remarks No-long-jump zone!!!
8446	*/
8447	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8448	{
8449	NOREF(pVM);
8450
8451	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8452	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8453
8454	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8455	{
8456	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8457	AssertRC(rc);
8458	}
8459
8460	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8461	{
8462	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8463	AssertRC(rc);
8464
8465	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8466	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8467	{
8468	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8469	AssertRC(rc);
8470	}
8471	}
8472
8473	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8474	{
8475	#if HC_ARCH_BITS == 64
8476	if (pVM->hm.s.fAllow64BitGuests)
8477	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8478	#endif
8479	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8480	}
8481
8482	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8483	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8484	{
8485	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8486	AssertRC(rc);
8487	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8488	}
8489
8490	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8491	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8492	}
8493
8494
8495	/**
8496	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8497	*
8498	* @param pVM Pointer to the VM.
8499	* @param pVCpu Pointer to the VMCPU.
8500	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8501	* out-of-sync. Make sure to update the required fields
8502	* before using them.
8503	*/
8504	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8505	{
8506	HMVMX_ASSERT_PREEMPT_SAFE();
8507
8508	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8509	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8510	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8511	#endif
8512
8513	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8514	{
8515	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8516	AssertRC(rc);
8517	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8518	}
8519	else if (HMCPU_CF_VALUE(pVCpu))
8520	{
8521	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8522	AssertRC(rc);
8523	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8524	}
8525
8526	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8527	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8528	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8529	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8530	}
8531
8532
8533	/**
8534	* Does the preparations before executing guest code in VT-x.
8535	*
8536	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8537	* recompiler/IEM. We must be cautious what we do here regarding committing
8538	* guest-state information into the VMCS assuming we assuredly execute the
8539	* guest in VT-x mode.
8540	*
8541	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8542	* the common-state (TRPM/forceflags), we must undo those changes so that the
8543	* recompiler/IEM can (and should) use them when it resumes guest execution.
8544	* Otherwise such operations must be done when we can no longer exit to ring-3.
8545	*
8546	* @returns Strict VBox status code.
8547	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8548	* have been disabled.
8549	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8550	* double-fault into the guest.
8551	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8552	* dispatched directly.
8553	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8554	*
8555	* @param pVM Pointer to the VM.
8556	* @param pVCpu Pointer to the VMCPU.
8557	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8558	* out-of-sync. Make sure to update the required fields
8559	* before using them.
8560	* @param pVmxTransient Pointer to the VMX transient structure.
8561	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8562	* us ignore some of the reasons for returning to
8563	* ring-3, and return VINF_EM_DBG_STEPPED if event
8564	* dispatching took place.
8565	*/
8566	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8567	{
8568	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8569
8570	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8571	PGMRZDynMapFlushAutoSet(pVCpu);
8572	#endif
8573
8574	/* Check force flag actions that might require us to go back to ring-3. */
8575	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8576	if (rc != VINF_SUCCESS)
8577	return rc;
8578
8579	#ifndef IEM_VERIFICATION_MODE_FULL
8580	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8581	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8582	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8583	{
8584	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8585	RTGCPHYS GCPhysApicBase;
8586	GCPhysApicBase = pMixedCtx->msrApicBase;
8587	GCPhysApicBase &= PAGE_BASE_GC_MASK;
8588
8589	/* Unalias any existing mapping. */
8590	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8591	AssertRCReturn(rc, rc);
8592
8593	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8594	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8595	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8596	AssertRCReturn(rc, rc);
8597
8598	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8599	}
8600	#endif /* !IEM_VERIFICATION_MODE_FULL */
8601
8602	if (TRPMHasTrap(pVCpu))
8603	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8604	else if (!pVCpu->hm.s.Event.fPending)
8605	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8606
8607	/*
8608	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8609	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8610	*/
8611	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8612	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8613	{
8614	Assert(rc == VINF_EM_RESET \|\| (rc == VINF_EM_DBG_STEPPED && fStepping));
8615	return rc;
8616	}
8617
8618	/*
8619	* Load the guest state bits, we can handle longjmps/getting preempted here.
8620	*
8621	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8622	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8623	* Hence, this needs to be done -after- injection of events.
8624	*/
8625	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8626
8627	/*
8628	* No longjmps to ring-3 from this point on!!!
8629	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8630	* This also disables flushing of the R0-logger instance (if any).
8631	*/
8632	VMMRZCallRing3Disable(pVCpu);
8633
8634	/*
8635	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8636	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8637	*
8638	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8639	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8640	*
8641	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8642	* executing guest code.
8643	*/
8644	pVmxTransient->uEflags = ASMIntDisableFlags();
8645	if ( ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8646	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8647	&& ( !fStepping /* Optimized for the non-stepping case, of course. */
8648	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
8649	{
8650	hmR0VmxClearEventVmcs(pVCpu);
8651	ASMSetFlags(pVmxTransient->uEflags);
8652	VMMRZCallRing3Enable(pVCpu);
8653	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8654	return VINF_EM_RAW_TO_R3;
8655	}
8656
8657	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8658	{
8659	hmR0VmxClearEventVmcs(pVCpu);
8660	ASMSetFlags(pVmxTransient->uEflags);
8661	VMMRZCallRing3Enable(pVCpu);
8662	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8663	return VINF_EM_RAW_INTERRUPT;
8664	}
8665
8666	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8667	pVCpu->hm.s.Event.fPending = false;
8668
8669	return VINF_SUCCESS;
8670	}
8671
8672
8673	/**
8674	* Prepares to run guest code in VT-x and we've committed to doing so. This
8675	* means there is no backing out to ring-3 or anywhere else at this
8676	* point.
8677	*
8678	* @param pVM Pointer to the VM.
8679	* @param pVCpu Pointer to the VMCPU.
8680	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8681	* out-of-sync. Make sure to update the required fields
8682	* before using them.
8683	* @param pVmxTransient Pointer to the VMX transient structure.
8684	*
8685	* @remarks Called with preemption disabled.
8686	* @remarks No-long-jump zone!!!
8687	*/
8688	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8689	{
8690	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8691	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8692	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8693
8694	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8695	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8696
8697	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8698	if (!CPUMIsGuestFPUStateActive(pVCpu))
8699	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8700	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8701	#endif
8702
8703	if ( pVCpu->hm.s.fPreloadGuestFpu
8704	&& !CPUMIsGuestFPUStateActive(pVCpu))
8705	{
8706	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8707	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8708	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8709	}
8710
8711	/*
8712	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
8713	*/
8714	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8715	&& pVCpu->hm.s.vmx.cMsrs > 0)
8716	{
8717	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8718	}
8719
8720	/*
8721	* Load the host state bits as we may've been preempted (only happens when
8722	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8723	*/
8724	/** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8725	* any effect to the host state needing to be saved? */
8726	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8727	{
8728	/* This ASSUMES that pfnStartVM has been set up already. */
8729	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8730	AssertRC(rc);
8731	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8732	}
8733	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8734
8735	/*
8736	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8737	*/
8738	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8739	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8740	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8741
8742	/* Store status of the shared guest-host state at the time of VM-entry. */
8743	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8744	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8745	{
8746	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8747	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8748	}
8749	else
8750	#endif
8751	{
8752	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8753	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8754	}
8755	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8756
8757	/*
8758	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8759	*/
8760	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8761	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8762
8763	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8764	RTCPUID idCurrentCpu = pCpu->idCpu;
8765	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8766	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8767	{
8768	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8769	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8770	}
8771
8772	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8773	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8774	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8775	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8776
8777	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8778
8779	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8780	to start executing. */
8781
8782	/*
8783	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8784	*/
8785	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8786	{
8787	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8788	{
8789	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8790	AssertRC(rc2);
8791	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8792	bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8793	true /* fUpdateHostMsr */);
8794	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8795	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8796	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8797	}
8798	else
8799	{
8800	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8801	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8802	}
8803	}
8804
8805	#ifdef VBOX_STRICT
8806	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8807	hmR0VmxCheckHostEferMsr(pVCpu);
8808	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8809	#endif
8810	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8811	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8812	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8813	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8814	#endif
8815	}
8816
8817
8818	/**
8819	* Performs some essential restoration of state after running guest code in
8820	* VT-x.
8821	*
8822	* @param pVM Pointer to the VM.
8823	* @param pVCpu Pointer to the VMCPU.
8824	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8825	* out-of-sync. Make sure to update the required fields
8826	* before using them.
8827	* @param pVmxTransient Pointer to the VMX transient structure.
8828	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8829	*
8830	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8831	*
8832	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8833	* unconditionally when it is safe to do so.
8834	*/
8835	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8836	{
8837	NOREF(pVM);
8838
8839	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8840
8841	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8842	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8843	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8844	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8845	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8846	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8847
8848	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8849	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8850
8851	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8852	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8853	Assert(!(ASMGetFlags() & X86_EFL_IF));
8854	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8855
8856	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8857	if (CPUMIsGuestFPUStateActive(pVCpu))
8858	{
8859	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8860	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8861	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8862	}
8863	#endif
8864
8865	#if HC_ARCH_BITS == 64
8866	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8867	#endif
8868	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8869	#ifdef VBOX_STRICT
8870	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8871	#endif
8872	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8873	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8874
8875	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8876	uint32_t uExitReason;
8877	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8878	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8879	AssertRC(rc);
8880	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8881	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8882
8883	/* Update the VM-exit history array. */
8884	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8885
8886	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8887	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8888	{
8889	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8890	pVmxTransient->fVMEntryFailed));
8891	return;
8892	}
8893
8894	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8895	{
8896	/** @todo We can optimize this by only syncing with our force-flags when
8897	* really needed and keeping the VMCS state as it is for most
8898	* VM-exits. */
8899	/* Update the guest interruptibility-state from the VMCS. */
8900	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8901
8902	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8903	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8904	AssertRC(rc);
8905	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8906	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8907	AssertRC(rc);
8908	#endif
8909
8910	/*
8911	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8912	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8913	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8914	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8915	*/
8916	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8917	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8918	{
8919	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8920	AssertRC(rc);
8921	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8922	}
8923	}
8924	}
8925
8926
8927	/**
8928	* Runs the guest code using VT-x the normal way.
8929	*
8930	* @returns VBox status code.
8931	* @param pVM Pointer to the VM.
8932	* @param pVCpu Pointer to the VMCPU.
8933	* @param pCtx Pointer to the guest-CPU context.
8934	*
8935	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8936	*/
8937	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8938	{
8939	VMXTRANSIENT VmxTransient;
8940	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8941	int rc = VERR_INTERNAL_ERROR_5;
8942	uint32_t cLoops = 0;
8943
8944	for (;; cLoops++)
8945	{
8946	Assert(!HMR0SuspendPending());
8947	HMVMX_ASSERT_CPU_SAFE();
8948
8949	/* Preparatory work for running guest code, this may force us to return
8950	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8951	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8952	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8953	if (rc != VINF_SUCCESS)
8954	break;
8955
8956	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8957	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8958	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8959
8960	/* Restore any residual host-state and save any bits shared between host
8961	and guest into the guest-CPU state. Re-enables interrupts! */
8962	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8963
8964	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8965	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8966	{
8967	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8968	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8969	return rc;
8970	}
8971
8972	/* Profile the VM-exit. */
8973	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8974	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8975	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8976	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8977	HMVMX_START_EXIT_DISPATCH_PROF();
8978
8979	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8980	if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8981	{
8982	hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8983	hmR0VmxSaveGuestState(pVCpu, pCtx);
8984	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8985	}
8986
8987	/* Handle the VM-exit. */
8988	#ifdef HMVMX_USE_FUNCTION_TABLE
8989	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8990	#else
8991	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8992	#endif
8993	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8994	if (rc != VINF_SUCCESS)
8995	break;
8996	if (cLoops > pVM->hm.s.cMaxResumeLoops)
8997	{
8998	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8999	rc = VINF_EM_RAW_INTERRUPT;
9000	break;
9001	}
9002	}
9003
9004	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9005	return rc;
9006	}
9007
9008
9009	/**
9010	* Single steps guest code using VT-x.
9011	*
9012	* @returns VBox status code.
9013	* @param pVM Pointer to the VM.
9014	* @param pVCpu Pointer to the VMCPU.
9015	* @param pCtx Pointer to the guest-CPU context.
9016	*
9017	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9018	*/
9019	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9020	{
9021	VMXTRANSIENT VmxTransient;
9022	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9023	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9024	uint32_t cLoops = 0;
9025	uint16_t uCsStart = pCtx->cs.Sel;
9026	uint64_t uRipStart = pCtx->rip;
9027
9028	for (;; cLoops++)
9029	{
9030	Assert(!HMR0SuspendPending());
9031	HMVMX_ASSERT_CPU_SAFE();
9032
9033	/* Preparatory work for running guest code, this may force us to return
9034	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9035	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9036	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
9037	if (rcStrict != VINF_SUCCESS)
9038	break;
9039
9040	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9041	rcStrict = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9042	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9043
9044	/* Restore any residual host-state and save any bits shared between host
9045	and guest into the guest-CPU state. Re-enables interrupts! */
9046	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
9047
9048	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9049	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
9050	{
9051	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9052	hmR0VmxReportWorldSwitchError(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict), pCtx, &VmxTransient);
9053	return VBOXSTRICTRC_TODO(rcStrict);
9054	}
9055
9056	/* Profile the VM-exit. */
9057	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9058	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9059	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9060	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9061	HMVMX_START_EXIT_DISPATCH_PROF();
9062
9063	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9064	if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
9065	{
9066	hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
9067	hmR0VmxSaveGuestState(pVCpu, pCtx);
9068	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
9069	}
9070
9071	/* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
9072	rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
9073	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9074	if (rcStrict != VINF_SUCCESS)
9075	break;
9076	if (cLoops > pVM->hm.s.cMaxResumeLoops)
9077	{
9078	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9079	rcStrict = VINF_EM_RAW_INTERRUPT;
9080	break;
9081	}
9082
9083	/*
9084	* Did the RIP change, if so, consider it a single step.
9085	* Otherwise, make sure one of the TFs gets set.
9086	*/
9087	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
9088	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
9089	AssertRCReturn(rc2, rc2);
9090	if ( pCtx->rip != uRipStart
9091	\|\| pCtx->cs.Sel != uCsStart)
9092	{
9093	rcStrict = VINF_EM_DBG_STEPPED;
9094	break;
9095	}
9096	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
9097	}
9098
9099	/*
9100	* Clear the X86_EFL_TF if necessary.
9101	*/
9102	if (pVCpu->hm.s.fClearTrapFlag)
9103	{
9104	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
9105	AssertRCReturn(rc2, rc2);
9106	pVCpu->hm.s.fClearTrapFlag = false;
9107	pCtx->eflags.Bits.u1TF = 0;
9108	}
9109	/** @todo there seems to be issues with the resume flag when the monitor trap
9110	* flag is pending without being used. Seen early in bios init when
9111	* accessing APIC page in protected mode. */
9112
9113	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9114	return VBOXSTRICTRC_TODO(rcStrict);
9115	}
9116
9117
9118	/**
9119	* Runs the guest code using VT-x.
9120	*
9121	* @returns VBox status code.
9122	* @param pVM Pointer to the VM.
9123	* @param pVCpu Pointer to the VMCPU.
9124	* @param pCtx Pointer to the guest-CPU context.
9125	*/
9126	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9127	{
9128	Assert(VMMRZCallRing3IsEnabled(pVCpu));
9129	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
9130	HMVMX_ASSERT_PREEMPT_SAFE();
9131
9132	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
9133
9134	int rc;
9135	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
9136	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
9137	else
9138	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
9139
9140	if (rc == VERR_EM_INTERPRETER)
9141	rc = VINF_EM_RAW_EMULATE_INSTR;
9142	else if (rc == VINF_EM_RESET)
9143	rc = VINF_EM_TRIPLE_FAULT;
9144
9145	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
9146	if (RT_FAILURE(rc2))
9147	{
9148	pVCpu->hm.s.u32HMError = rc;
9149	rc = rc2;
9150	}
9151	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
9152	return rc;
9153	}
9154
9155
9156	#ifndef HMVMX_USE_FUNCTION_TABLE
9157	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
9158	{
9159	#ifdef DEBUG_ramshankar
9160	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9161	# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9162	#endif
9163	int rc;
9164	switch (rcReason)
9165	{
9166	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9167	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9168	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9169	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9170	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9171	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9172	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9173	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9174	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9175	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9176	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9177	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9178	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9179	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9180	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9181	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9182	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9183	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9184	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9185	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9186	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9187	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9188	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9189	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9190	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9191	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9192	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9193	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9194	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9195	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9196	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9197	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9198	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9199	case VMX_EXIT_VMCALL: /* SVVMCS(); / rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
9200
9201	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9202	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9203	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9204	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9205	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9206	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9207	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9208	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9209	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9210
9211	case VMX_EXIT_VMCLEAR:
9212	case VMX_EXIT_VMLAUNCH:
9213	case VMX_EXIT_VMPTRLD:
9214	case VMX_EXIT_VMPTRST:
9215	case VMX_EXIT_VMREAD:
9216	case VMX_EXIT_VMRESUME:
9217	case VMX_EXIT_VMWRITE:
9218	case VMX_EXIT_VMXOFF:
9219	case VMX_EXIT_VMXON:
9220	case VMX_EXIT_INVEPT:
9221	case VMX_EXIT_INVVPID:
9222	case VMX_EXIT_VMFUNC:
9223	case VMX_EXIT_XSAVES:
9224	case VMX_EXIT_XRSTORS:
9225	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9226	break;
9227	case VMX_EXIT_RESERVED_60:
9228	case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
9229	case VMX_EXIT_RESERVED_62:
9230	default:
9231	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9232	break;
9233	}
9234	return rc;
9235	}
9236	#endif /* !HMVMX_USE_FUNCTION_TABLE */
9237
9238
9239	/**
9240	* Single-stepping VM-exit filtering.
9241	*
9242	* This is preprocessing the exits and deciding whether we've gotten far enough
9243	* to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9244	* performed.
9245	*
9246	* @returns Strict VBox status code.
9247	* @param pVCpu The virtual CPU of the calling EMT.
9248	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9249	* out-of-sync. Make sure to update the required
9250	* fields before using them.
9251	* @param pVmxTransient Pointer to the VMX-transient structure.
9252	* @param uExitReason The VM-exit reason.
9253	*/
9254	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9255	uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9256	{
9257	switch (uExitReason)
9258	{
9259	case VMX_EXIT_XCPT_OR_NMI:
9260	{
9261	/* Check for host NMI. */
9262	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9263	AssertRCReturn(rc2, rc2);
9264	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9265	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9266	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9267	/* fall thru */
9268	}
9269
9270	case VMX_EXIT_EPT_MISCONFIG:
9271	case VMX_EXIT_TRIPLE_FAULT:
9272	case VMX_EXIT_APIC_ACCESS:
9273	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9274	case VMX_EXIT_TASK_SWITCH:
9275
9276	/* Instruction specific VM-exits: */
9277	case VMX_EXIT_IO_INSTR:
9278	case VMX_EXIT_CPUID:
9279	case VMX_EXIT_RDTSC:
9280	case VMX_EXIT_RDTSCP:
9281	case VMX_EXIT_MOV_CRX:
9282	case VMX_EXIT_MWAIT:
9283	case VMX_EXIT_MONITOR:
9284	case VMX_EXIT_RDMSR:
9285	case VMX_EXIT_WRMSR:
9286	case VMX_EXIT_MOV_DRX:
9287	case VMX_EXIT_HLT:
9288	case VMX_EXIT_INVD:
9289	case VMX_EXIT_INVLPG:
9290	case VMX_EXIT_RSM:
9291	case VMX_EXIT_PAUSE:
9292	case VMX_EXIT_XDTR_ACCESS:
9293	case VMX_EXIT_TR_ACCESS:
9294	case VMX_EXIT_WBINVD:
9295	case VMX_EXIT_XSETBV:
9296	case VMX_EXIT_RDRAND:
9297	case VMX_EXIT_INVPCID:
9298	case VMX_EXIT_GETSEC:
9299	case VMX_EXIT_RDPMC:
9300	case VMX_EXIT_VMCALL:
9301	case VMX_EXIT_VMCLEAR:
9302	case VMX_EXIT_VMLAUNCH:
9303	case VMX_EXIT_VMPTRLD:
9304	case VMX_EXIT_VMPTRST:
9305	case VMX_EXIT_VMREAD:
9306	case VMX_EXIT_VMRESUME:
9307	case VMX_EXIT_VMWRITE:
9308	case VMX_EXIT_VMXOFF:
9309	case VMX_EXIT_VMXON:
9310	case VMX_EXIT_INVEPT:
9311	case VMX_EXIT_INVVPID:
9312	case VMX_EXIT_VMFUNC:
9313	{
9314	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9315	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9316	AssertRCReturn(rc2, rc2);
9317	if ( pMixedCtx->rip != uRipStart
9318	\|\| pMixedCtx->cs.Sel != uCsStart)
9319	return VINF_EM_DBG_STEPPED;
9320	break;
9321	}
9322	}
9323
9324	/*
9325	* Normal processing.
9326	*/
9327	#ifdef HMVMX_USE_FUNCTION_TABLE
9328	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9329	#else
9330	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9331	#endif
9332	}
9333
9334
9335	#ifdef DEBUG
9336	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9337	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9338	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9339
9340	# define HMVMX_ASSERT_PREEMPT_CPUID() \
9341	do \
9342	{ \
9343	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9344	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9345	} while (0)
9346
9347	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9348	do { \
9349	AssertPtr(pVCpu); \
9350	AssertPtr(pMixedCtx); \
9351	AssertPtr(pVmxTransient); \
9352	Assert(pVmxTransient->fVMEntryFailed == false); \
9353	Assert(ASMIntAreEnabled()); \
9354	HMVMX_ASSERT_PREEMPT_SAFE(); \
9355	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9356	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9357	HMVMX_ASSERT_PREEMPT_SAFE(); \
9358	if (VMMR0IsLogFlushDisabled(pVCpu)) \
9359	HMVMX_ASSERT_PREEMPT_CPUID(); \
9360	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9361	} while (0)
9362
9363	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9364	do { \
9365	Log4Func(("\n")); \
9366	} while (0)
9367	#else /* Release builds */
9368	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9369	do { \
9370	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9371	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9372	} while (0)
9373	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9374	#endif
9375
9376
9377	/**
9378	* Advances the guest RIP after reading it from the VMCS.
9379	*
9380	* @returns VBox status code.
9381	* @param pVCpu Pointer to the VMCPU.
9382	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9383	* out-of-sync. Make sure to update the required fields
9384	* before using them.
9385	* @param pVmxTransient Pointer to the VMX transient structure.
9386	*
9387	* @remarks No-long-jump zone!!!
9388	*/
9389	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9390	{
9391	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9392	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9393	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9394	AssertRCReturn(rc, rc);
9395
9396	pMixedCtx->rip += pVmxTransient->cbInstr;
9397	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9398
9399	/*
9400	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9401	* pending debug exception field as it takes care of priority of events.
9402	*
9403	* See Intel spec. 32.2.1 "Debug Exceptions".
9404	*/
9405	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9406
9407	return rc;
9408	}
9409
9410
9411	/**
9412	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9413	* and update error record fields accordingly.
9414	*
9415	* @return VMX_IGS_* return codes.
9416	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9417	* wrong with the guest state.
9418	*
9419	* @param pVM Pointer to the VM.
9420	* @param pVCpu Pointer to the VMCPU.
9421	* @param pCtx Pointer to the guest-CPU state.
9422	*
9423	* @remarks This function assumes our cache of the VMCS controls
9424	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9425	*/
9426	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9427	{
9428	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9429	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9430	uError = (err); \
9431	break; \
9432	} else do { } while (0)
9433
9434	int rc;
9435	uint32_t uError = VMX_IGS_ERROR;
9436	uint32_t u32Val;
9437	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9438
9439	do
9440	{
9441	/*
9442	* CR0.
9443	*/
9444	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9445	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9446	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9447	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9448	if (fUnrestrictedGuest)
9449	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9450
9451	uint32_t u32GuestCR0;
9452	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9453	AssertRCBreak(rc);
9454	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9455	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9456	if ( !fUnrestrictedGuest
9457	&& (u32GuestCR0 & X86_CR0_PG)
9458	&& !(u32GuestCR0 & X86_CR0_PE))
9459	{
9460	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9461	}
9462
9463	/*
9464	* CR4.
9465	*/
9466	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9467	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9468
9469	uint32_t u32GuestCR4;
9470	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9471	AssertRCBreak(rc);
9472	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9473	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9474
9475	/*
9476	* IA32_DEBUGCTL MSR.
9477	*/
9478	uint64_t u64Val;
9479	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9480	AssertRCBreak(rc);
9481	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9482	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9483	{
9484	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9485	}
9486	uint64_t u64DebugCtlMsr = u64Val;
9487
9488	#ifdef VBOX_STRICT
9489	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9490	AssertRCBreak(rc);
9491	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9492	#endif
9493	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9494
9495	/*
9496	* RIP and RFLAGS.
9497	*/
9498	uint32_t u32Eflags;
9499	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9500	if (HMVMX_IS_64BIT_HOST_MODE())
9501	{
9502	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9503	AssertRCBreak(rc);
9504	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9505	if ( !fLongModeGuest
9506	\|\| !pCtx->cs.Attr.n.u1Long)
9507	{
9508	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9509	}
9510	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9511	* must be identical if the "IA-32e mode guest" VM-entry
9512	* control is 1 and CS.L is 1. No check applies if the
9513	* CPU supports 64 linear-address bits. */
9514
9515	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9516	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9517	AssertRCBreak(rc);
9518	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9519	VMX_IGS_RFLAGS_RESERVED);
9520	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9521	u32Eflags = u64Val;
9522	}
9523	else
9524	#endif
9525	{
9526	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9527	AssertRCBreak(rc);
9528	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9529	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9530	}
9531
9532	if ( fLongModeGuest
9533	\|\| ( fUnrestrictedGuest
9534	&& !(u32GuestCR0 & X86_CR0_PE)))
9535	{
9536	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9537	}
9538
9539	uint32_t u32EntryInfo;
9540	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9541	AssertRCBreak(rc);
9542	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9543	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9544	{
9545	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9546	}
9547
9548	/*
9549	* 64-bit checks.
9550	*/
9551	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9552	if (HMVMX_IS_64BIT_HOST_MODE())
9553	{
9554	if (fLongModeGuest)
9555	{
9556	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9557	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9558	}
9559
9560	if ( !fLongModeGuest
9561	&& (u32GuestCR4 & X86_CR4_PCIDE))
9562	{
9563	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9564	}
9565
9566	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9567	* 51:32 beyond the processor's physical-address width are 0. */
9568
9569	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9570	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9571	{
9572	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9573	}
9574
9575	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9576	AssertRCBreak(rc);
9577	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9578
9579	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9580	AssertRCBreak(rc);
9581	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9582	}
9583	#endif
9584
9585	/*
9586	* PERF_GLOBAL MSR.
9587	*/
9588	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9589	{
9590	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9591	AssertRCBreak(rc);
9592	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9593	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9594	}
9595
9596	/*
9597	* PAT MSR.
9598	*/
9599	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9600	{
9601	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9602	AssertRCBreak(rc);
9603	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9604	for (unsigned i = 0; i < 8; i++)
9605	{
9606	uint8_t u8Val = (u64Val & 0xff);
9607	if ( u8Val != 0 /* UC */
9608	&& u8Val != 1 /* WC */
9609	&& u8Val != 4 /* WT */
9610	&& u8Val != 5 /* WP */
9611	&& u8Val != 6 /* WB */
9612	&& u8Val != 7 /* UC- */)
9613	{
9614	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9615	}
9616	u64Val >>= 8;
9617	}
9618	}
9619
9620	/*
9621	* EFER MSR.
9622	*/
9623	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9624	{
9625	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9626	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9627	AssertRCBreak(rc);
9628	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9629	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9630	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9631	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9632	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9633	\|\| !(u32GuestCR0 & X86_CR0_PG)
9634	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9635	VMX_IGS_EFER_LMA_LME_MISMATCH);
9636	}
9637
9638	/*
9639	* Segment registers.
9640	*/
9641	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9642	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9643	if (!(u32Eflags & X86_EFL_VM))
9644	{
9645	/* CS */
9646	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9647	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9648	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9649	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9650	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9651	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9652	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9653	/* CS cannot be loaded with NULL in protected mode. */
9654	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9655	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9656	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9657	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9658	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9659	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9660	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9661	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9662	else
9663	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9664
9665	/* SS */
9666	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9667	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9668	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9669	if ( !(pCtx->cr0 & X86_CR0_PE)
9670	\|\| pCtx->cs.Attr.n.u4Type == 3)
9671	{
9672	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9673	}
9674	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9675	{
9676	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9677	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9678	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9679	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9680	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9681	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9682	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9683	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9684	}
9685
9686	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9687	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9688	{
9689	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9690	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9691	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9692	\|\| pCtx->ds.Attr.n.u4Type > 11
9693	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9694	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9695	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9696	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9697	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9698	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9699	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9700	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9701	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9702	}
9703	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9704	{
9705	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9706	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9707	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9708	\|\| pCtx->es.Attr.n.u4Type > 11
9709	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9710	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9711	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9712	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9713	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9714	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9715	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9716	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9717	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9718	}
9719	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9720	{
9721	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9722	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9723	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9724	\|\| pCtx->fs.Attr.n.u4Type > 11
9725	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9726	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9727	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9728	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9729	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9730	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9731	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9732	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9733	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9734	}
9735	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9736	{
9737	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9738	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9739	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9740	\|\| pCtx->gs.Attr.n.u4Type > 11
9741	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9742	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9743	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9744	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9745	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9746	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9747	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9748	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9749	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9750	}
9751	/* 64-bit capable CPUs. */
9752	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9753	if (HMVMX_IS_64BIT_HOST_MODE())
9754	{
9755	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9756	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9757	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9758	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9759	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9760	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9761	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9762	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9763	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9764	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9765	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9766	}
9767	#endif
9768	}
9769	else
9770	{
9771	/* V86 mode checks. */
9772	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9773	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9774	{
9775	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9776	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9777	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9778	}
9779	else
9780	{
9781	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9782	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9783	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9784	}
9785
9786	/* CS */
9787	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9788	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9789	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9790	/* SS */
9791	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9792	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9793	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9794	/* DS */
9795	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9796	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9797	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9798	/* ES */
9799	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9800	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9801	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9802	/* FS */
9803	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9804	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9805	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9806	/* GS */
9807	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9808	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9809	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9810	/* 64-bit capable CPUs. */
9811	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9812	if (HMVMX_IS_64BIT_HOST_MODE())
9813	{
9814	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9815	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9816	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9817	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9818	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9819	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9820	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9821	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9822	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9823	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9824	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9825	}
9826	#endif
9827	}
9828
9829	/*
9830	* TR.
9831	*/
9832	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9833	/* 64-bit capable CPUs. */
9834	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9835	if (HMVMX_IS_64BIT_HOST_MODE())
9836	{
9837	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9838	}
9839	#endif
9840	if (fLongModeGuest)
9841	{
9842	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9843	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9844	}
9845	else
9846	{
9847	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9848	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9849	VMX_IGS_TR_ATTR_TYPE_INVALID);
9850	}
9851	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9852	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9853	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9854	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9855	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9856	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9857	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9858	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9859
9860	/*
9861	* GDTR and IDTR.
9862	*/
9863	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9864	if (HMVMX_IS_64BIT_HOST_MODE())
9865	{
9866	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9867	AssertRCBreak(rc);
9868	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9869
9870	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9871	AssertRCBreak(rc);
9872	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9873	}
9874	#endif
9875
9876	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9877	AssertRCBreak(rc);
9878	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9879
9880	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9881	AssertRCBreak(rc);
9882	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9883
9884	/*
9885	* Guest Non-Register State.
9886	*/
9887	/* Activity State. */
9888	uint32_t u32ActivityState;
9889	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9890	AssertRCBreak(rc);
9891	HMVMX_CHECK_BREAK( !u32ActivityState
9892	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9893	VMX_IGS_ACTIVITY_STATE_INVALID);
9894	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9895	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9896	uint32_t u32IntrState;
9897	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9898	AssertRCBreak(rc);
9899	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9900	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9901	{
9902	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9903	}
9904
9905	/** @todo Activity state and injecting interrupts. Left as a todo since we
9906	* currently don't use activity states but ACTIVE. */
9907
9908	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9909	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9910
9911	/* Guest interruptibility-state. */
9912	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9913	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9914	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9915	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9916	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9917	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9918	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9919	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9920	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9921	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9922	{
9923	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9924	{
9925	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9926	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9927	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9928	}
9929	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9930	{
9931	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9932	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9933	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9934	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9935	}
9936	}
9937	/** @todo Assumes the processor is not in SMM. */
9938	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9939	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9940	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9941	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9942	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9943	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9944	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9945	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9946	{
9947	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9948	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9949	}
9950
9951	/* Pending debug exceptions. */
9952	if (HMVMX_IS_64BIT_HOST_MODE())
9953	{
9954	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9955	AssertRCBreak(rc);
9956	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9957	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9958	u32Val = u64Val; /* For pending debug exceptions checks below. */
9959	}
9960	else
9961	{
9962	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9963	AssertRCBreak(rc);
9964	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9965	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9966	}
9967
9968	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9969	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9970	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9971	{
9972	if ( (u32Eflags & X86_EFL_TF)
9973	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9974	{
9975	/* Bit 14 is PendingDebug.BS. */
9976	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9977	}
9978	if ( !(u32Eflags & X86_EFL_TF)
9979	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9980	{
9981	/* Bit 14 is PendingDebug.BS. */
9982	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9983	}
9984	}
9985
9986	/* VMCS link pointer. */
9987	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9988	AssertRCBreak(rc);
9989	if (u64Val != UINT64_C(0xffffffffffffffff))
9990	{
9991	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9992	/** @todo Bits beyond the processor's physical-address width MBZ. */
9993	/** @todo 32-bit located in memory referenced by value of this field (as a
9994	* physical address) must contain the processor's VMCS revision ID. */
9995	/** @todo SMM checks. */
9996	}
9997
9998	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9999	* not using Nested Paging? */
10000	if ( pVM->hm.s.fNestedPaging
10001	&& !fLongModeGuest
10002	&& CPUMIsGuestInPAEModeEx(pCtx))
10003	{
10004	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10005	AssertRCBreak(rc);
10006	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10007
10008	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10009	AssertRCBreak(rc);
10010	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10011
10012	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10013	AssertRCBreak(rc);
10014	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10015
10016	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10017	AssertRCBreak(rc);
10018	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10019	}
10020
10021	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10022	if (uError == VMX_IGS_ERROR)
10023	uError = VMX_IGS_REASON_NOT_FOUND;
10024	} while (0);
10025
10026	pVCpu->hm.s.u32HMError = uError;
10027	return uError;
10028
10029	#undef HMVMX_ERROR_BREAK
10030	#undef HMVMX_CHECK_BREAK
10031	}
10032
10033	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10034	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10035	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10036
10037	/** @name VM-exit handlers.
10038	* @{
10039	*/
10040
10041	/**
10042	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10043	*/
10044	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10045	{
10046	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10047	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10048	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10049	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
10050	return VINF_SUCCESS;
10051	return VINF_EM_RAW_INTERRUPT;
10052	}
10053
10054
10055	/**
10056	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10057	*/
10058	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10059	{
10060	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10061	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10062
10063	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10064	AssertRCReturn(rc, rc);
10065
10066	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10067	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10068	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10069	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10070
10071	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10072	{
10073	/*
10074	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
10075	* anything we inject is not going to cause a VM-exit directly for the event being injected.
10076	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10077	*
10078	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
10079	*/
10080	VMXDispatchHostNmi();
10081	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10082	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10083	return VINF_SUCCESS;
10084	}
10085
10086	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10087	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10088	if (RT_UNLIKELY(rc != VINF_SUCCESS))
10089	{
10090	if (rc == VINF_HM_DOUBLE_FAULT)
10091	rc = VINF_SUCCESS;
10092	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10093	return rc;
10094	}
10095
10096	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10097	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10098	switch (uIntType)
10099	{
10100	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10101	Assert(uVector == X86_XCPT_DB);
10102	/* no break */
10103	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10104	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10105	/* no break */
10106	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10107	{
10108	switch (uVector)
10109	{
10110	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
10111	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
10112	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
10113	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
10114	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
10115	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
10116	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10117	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10118	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10119	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10120	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10121	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10122	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10123	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10124	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10125	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10126	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10127	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
10128	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10129	#endif
10130	default:
10131	{
10132	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10133	AssertRCReturn(rc, rc);
10134
10135	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10136	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10137	{
10138	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10139	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10140	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10141
10142	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10143	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10144	AssertRCReturn(rc, rc);
10145	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10146	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10147	0 /* GCPtrFaultAddress */);
10148	AssertRCReturn(rc, rc);
10149	}
10150	else
10151	{
10152	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10153	pVCpu->hm.s.u32HMError = uVector;
10154	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10155	}
10156	break;
10157	}
10158	}
10159	break;
10160	}
10161
10162	default:
10163	{
10164	pVCpu->hm.s.u32HMError = uExitIntInfo;
10165	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10166	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10167	break;
10168	}
10169	}
10170	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10171	return rc;
10172	}
10173
10174
10175	/**
10176	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10177	*/
10178	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10179	{
10180	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10181
10182	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
10183	hmR0VmxClearIntWindowExitVmcs(pVCpu);
10184
10185	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10186	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
10187	return VINF_SUCCESS;
10188	}
10189
10190
10191	/**
10192	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
10193	*/
10194	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10195	{
10196	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10197	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
10198	{
10199	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
10200	HMVMX_RETURN_UNEXPECTED_EXIT();
10201	}
10202
10203	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
10204
10205	/*
10206	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
10207	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
10208	*/
10209	uint32_t uIntrState = 0;
10210	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10211	AssertRCReturn(rc, rc);
10212
10213	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
10214	if ( fBlockSti
10215	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
10216	{
10217	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10218	}
10219
10220	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
10221	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
10222
10223	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10224	return VINF_SUCCESS;
10225	}
10226
10227
10228	/**
10229	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10230	*/
10231	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10232	{
10233	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10234	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
10235	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10236	}
10237
10238
10239	/**
10240	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10241	*/
10242	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10243	{
10244	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10245	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10246	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10247	}
10248
10249
10250	/**
10251	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10252	*/
10253	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10254	{
10255	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10256	PVM pVM = pVCpu->CTX_SUFF(pVM);
10257	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10258	if (RT_LIKELY(rc == VINF_SUCCESS))
10259	{
10260	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10261	Assert(pVmxTransient->cbInstr == 2);
10262	}
10263	else
10264	{
10265	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10266	rc = VERR_EM_INTERPRETER;
10267	}
10268	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10269	return rc;
10270	}
10271
10272
10273	/**
10274	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10275	*/
10276	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10277	{
10278	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10279	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10280	AssertRCReturn(rc, rc);
10281
10282	if (pMixedCtx->cr4 & X86_CR4_SMXE)
10283	return VINF_EM_RAW_EMULATE_INSTR;
10284
10285	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10286	HMVMX_RETURN_UNEXPECTED_EXIT();
10287	}
10288
10289
10290	/**
10291	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10292	*/
10293	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10294	{
10295	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10296	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10297	AssertRCReturn(rc, rc);
10298
10299	PVM pVM = pVCpu->CTX_SUFF(pVM);
10300	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10301	if (RT_LIKELY(rc == VINF_SUCCESS))
10302	{
10303	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10304	Assert(pVmxTransient->cbInstr == 2);
10305	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10306	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10307	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10308	}
10309	else
10310	rc = VERR_EM_INTERPRETER;
10311	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10312	return rc;
10313	}
10314
10315
10316	/**
10317	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10318	*/
10319	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10320	{
10321	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10322	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10323	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10324	AssertRCReturn(rc, rc);
10325
10326	PVM pVM = pVCpu->CTX_SUFF(pVM);
10327	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10328	if (RT_LIKELY(rc == VINF_SUCCESS))
10329	{
10330	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10331	Assert(pVmxTransient->cbInstr == 3);
10332	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10333	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10334	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10335	}
10336	else
10337	{
10338	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10339	rc = VERR_EM_INTERPRETER;
10340	}
10341	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10342	return rc;
10343	}
10344
10345
10346	/**
10347	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10348	*/
10349	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10350	{
10351	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10352	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10353	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10354	AssertRCReturn(rc, rc);
10355
10356	PVM pVM = pVCpu->CTX_SUFF(pVM);
10357	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10358	if (RT_LIKELY(rc == VINF_SUCCESS))
10359	{
10360	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10361	Assert(pVmxTransient->cbInstr == 2);
10362	}
10363	else
10364	{
10365	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10366	rc = VERR_EM_INTERPRETER;
10367	}
10368	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10369	return rc;
10370	}
10371
10372
10373	/**
10374	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10375	*/
10376	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10377	{
10378	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10380
10381	if (pVCpu->hm.s.fHypercallsEnabled)
10382	{
10383	#if 0
10384	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10385	AssertRCReturn(rc, rc);
10386	#else
10387	/* Aggressive state sync. for now. */
10388	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10389	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
10390	#endif
10391	AssertRCReturn(rc, rc);
10392
10393	rc = GIMHypercall(pVCpu, pMixedCtx);
10394	if (RT_SUCCESS(rc))
10395	{
10396	/* If the hypercall changes anything other than guest general-purpose registers,
10397	we would need to reload the guest changed bits here before VM-reentry. */
10398	hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10399	return VINF_SUCCESS;
10400	}
10401	}
10402
10403	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10404	return VINF_SUCCESS;
10405	}
10406
10407
10408	/**
10409	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10410	*/
10411	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10412	{
10413	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10414	PVM pVM = pVCpu->CTX_SUFF(pVM);
10415	Assert(!pVM->hm.s.fNestedPaging);
10416
10417	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10418	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10419	AssertRCReturn(rc, rc);
10420
10421	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10422	rc = VBOXSTRICTRC_VAL(rc2);
10423	if (RT_LIKELY(rc == VINF_SUCCESS))
10424	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10425	else
10426	{
10427	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10428	pVmxTransient->uExitQualification, rc));
10429	}
10430	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10431	return rc;
10432	}
10433
10434
10435	/**
10436	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10437	*/
10438	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10439	{
10440	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10441	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10442	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10443	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10444	AssertRCReturn(rc, rc);
10445
10446	PVM pVM = pVCpu->CTX_SUFF(pVM);
10447	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10448	if (RT_LIKELY(rc == VINF_SUCCESS))
10449	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10450	else
10451	{
10452	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10453	rc = VERR_EM_INTERPRETER;
10454	}
10455	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10456	return rc;
10457	}
10458
10459
10460	/**
10461	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10462	*/
10463	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10464	{
10465	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10466	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10467	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10468	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10469	AssertRCReturn(rc, rc);
10470
10471	PVM pVM = pVCpu->CTX_SUFF(pVM);
10472	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10473	rc = VBOXSTRICTRC_VAL(rc2);
10474	if (RT_LIKELY( rc == VINF_SUCCESS
10475	\|\| rc == VINF_EM_HALT))
10476	{
10477	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10478	AssertRCReturn(rc3, rc3);
10479
10480	if ( rc == VINF_EM_HALT
10481	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10482	{
10483	rc = VINF_SUCCESS;
10484	}
10485	}
10486	else
10487	{
10488	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10489	rc = VERR_EM_INTERPRETER;
10490	}
10491	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
10492	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10493	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10494	return rc;
10495	}
10496
10497
10498	/**
10499	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10500	*/
10501	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10502	{
10503	/*
10504	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10505	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10506	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
10507	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10508	*/
10509	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10510	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10511	HMVMX_RETURN_UNEXPECTED_EXIT();
10512	}
10513
10514
10515	/**
10516	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10517	*/
10518	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10519	{
10520	/*
10521	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10522	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10523	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
10524	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10525	*/
10526	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10527	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10528	HMVMX_RETURN_UNEXPECTED_EXIT();
10529	}
10530
10531
10532	/**
10533	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10534	*/
10535	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10536	{
10537	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10538	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10539	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10540	HMVMX_RETURN_UNEXPECTED_EXIT();
10541	}
10542
10543
10544	/**
10545	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10546	*/
10547	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10548	{
10549	/*
10550	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10551	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10552	* See Intel spec. 25.3 "Other Causes of VM-exits".
10553	*/
10554	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10555	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10556	HMVMX_RETURN_UNEXPECTED_EXIT();
10557	}
10558
10559
10560	/**
10561	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10562	* VM-exit.
10563	*/
10564	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10565	{
10566	/*
10567	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10568	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10569	*
10570	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10571	* See Intel spec. "23.8 Restrictions on VMX operation".
10572	*/
10573	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10574	return VINF_SUCCESS;
10575	}
10576
10577
10578	/**
10579	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10580	* VM-exit.
10581	*/
10582	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10583	{
10584	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10585	return VINF_EM_RESET;
10586	}
10587
10588
10589	/**
10590	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10591	*/
10592	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10593	{
10594	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10595	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10596	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10597	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10598	AssertRCReturn(rc, rc);
10599
10600	pMixedCtx->rip++;
10601	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10602	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10603	rc = VINF_SUCCESS;
10604	else
10605	rc = VINF_EM_HALT;
10606
10607	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10608	if (rc != VINF_SUCCESS)
10609	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
10610	return rc;
10611	}
10612
10613
10614	/**
10615	* VM-exit handler for instructions that result in a #UD exception delivered to
10616	* the guest.
10617	*/
10618	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10619	{
10620	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10621	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10622	return VINF_SUCCESS;
10623	}
10624
10625
10626	/**
10627	* VM-exit handler for expiry of the VMX preemption timer.
10628	*/
10629	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10630	{
10631	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10632
10633	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10634	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10635
10636	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10637	PVM pVM = pVCpu->CTX_SUFF(pVM);
10638	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10639	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10640	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10641	}
10642
10643
10644	/**
10645	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10646	*/
10647	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10648	{
10649	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10650
10651	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10652	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
10653	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10654	AssertRCReturn(rc, rc);
10655
10656	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
10657	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
10658
10659	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
10660
10661	return VBOXSTRICTRC_TODO(rcStrict);
10662	}
10663
10664
10665	/**
10666	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10667	*/
10668	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10669	{
10670	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10671
10672	/* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
10673	/** @todo implement EMInterpretInvpcid() */
10674	return VERR_EM_INTERPRETER;
10675	}
10676
10677
10678	/**
10679	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10680	* Error VM-exit.
10681	*/
10682	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10683	{
10684	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10685	AssertRCReturn(rc, rc);
10686
10687	rc = hmR0VmxCheckVmcsCtls(pVCpu);
10688	AssertRCReturn(rc, rc);
10689
10690	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10691	NOREF(uInvalidReason);
10692
10693	#ifdef VBOX_STRICT
10694	uint32_t uIntrState;
10695	HMVMXHCUINTREG uHCReg;
10696	uint64_t u64Val;
10697	uint32_t u32Val;
10698
10699	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10700	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10701	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10702	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10703	AssertRCReturn(rc, rc);
10704
10705	Log4(("uInvalidReason %u\n", uInvalidReason));
10706	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10707	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10708	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10709	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10710
10711	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10712	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10713	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10714	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10715	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10716	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10717	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10718	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10719	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10720	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10721	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10722	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10723	#else
10724	NOREF(pVmxTransient);
10725	#endif
10726
10727	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10728	return VERR_VMX_INVALID_GUEST_STATE;
10729	}
10730
10731
10732	/**
10733	* VM-exit handler for VM-entry failure due to an MSR-load
10734	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10735	*/
10736	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10737	{
10738	NOREF(pVmxTransient);
10739	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10740	HMVMX_RETURN_UNEXPECTED_EXIT();
10741	}
10742
10743
10744	/**
10745	* VM-exit handler for VM-entry failure due to a machine-check event
10746	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10747	*/
10748	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10749	{
10750	NOREF(pVmxTransient);
10751	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10752	HMVMX_RETURN_UNEXPECTED_EXIT();
10753	}
10754
10755
10756	/**
10757	* VM-exit handler for all undefined reasons. Should never ever happen.. in
10758	* theory.
10759	*/
10760	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10761	{
10762	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10763	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10764	return VERR_VMX_UNDEFINED_EXIT_CODE;
10765	}
10766
10767
10768	/**
10769	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10770	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10771	* Conditional VM-exit.
10772	*/
10773	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10774	{
10775	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10776
10777	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10778	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10779	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10780	return VERR_EM_INTERPRETER;
10781	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10782	HMVMX_RETURN_UNEXPECTED_EXIT();
10783	}
10784
10785
10786	/**
10787	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10788	*/
10789	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10790	{
10791	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10792
10793	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10794	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10795	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10796	return VERR_EM_INTERPRETER;
10797	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10798	HMVMX_RETURN_UNEXPECTED_EXIT();
10799	}
10800
10801
10802	/**
10803	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10804	*/
10805	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10806	{
10807	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10808
10809	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10810	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10811	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10812	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10813	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10814	{
10815	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10816	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10817	}
10818	AssertRCReturn(rc, rc);
10819	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10820
10821	#ifdef VBOX_STRICT
10822	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10823	{
10824	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10825	&& pMixedCtx->ecx != MSR_K6_EFER)
10826	{
10827	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10828	HMVMX_RETURN_UNEXPECTED_EXIT();
10829	}
10830	# if HC_ARCH_BITS == 64
10831	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10832	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10833	{
10834	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10835	HMVMX_RETURN_UNEXPECTED_EXIT();
10836	}
10837	# endif
10838	}
10839	#endif
10840
10841	PVM pVM = pVCpu->CTX_SUFF(pVM);
10842	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10843	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
10844	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10845	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10846	if (RT_LIKELY(rc == VINF_SUCCESS))
10847	{
10848	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10849	Assert(pVmxTransient->cbInstr == 2);
10850	}
10851	return rc;
10852	}
10853
10854
10855	/**
10856	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10857	*/
10858	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10859	{
10860	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10861	PVM pVM = pVCpu->CTX_SUFF(pVM);
10862	int rc = VINF_SUCCESS;
10863
10864	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10865	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10866	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10867	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10868	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10869	{
10870	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10871	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10872	}
10873	AssertRCReturn(rc, rc);
10874	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10875
10876	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10877	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10878	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10879
10880	if (RT_LIKELY(rc == VINF_SUCCESS))
10881	{
10882	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10883
10884	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
10885	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10886	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10887	{
10888	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10889	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10890	EMInterpretWrmsr() changes it. */
10891	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10892	}
10893	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10894	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10895	else if (pMixedCtx->ecx == MSR_K6_EFER)
10896	{
10897	/*
10898	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10899	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10900	* the other bits as well, SCE and NXE. See @bugref{7368}.
10901	*/
10902	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
10903	}
10904
10905	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10906	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10907	{
10908	switch (pMixedCtx->ecx)
10909	{
10910	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10911	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10912	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10913	case MSR_K8_FS_BASE: /* no break */
10914	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10915	case MSR_K6_EFER: /* already handled above */ break;
10916	default:
10917	{
10918	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10919	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10920	#if HC_ARCH_BITS == 64
10921	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10922	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10923	#endif
10924	break;
10925	}
10926	}
10927	}
10928	#ifdef VBOX_STRICT
10929	else
10930	{
10931	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10932	switch (pMixedCtx->ecx)
10933	{
10934	case MSR_IA32_SYSENTER_CS:
10935	case MSR_IA32_SYSENTER_EIP:
10936	case MSR_IA32_SYSENTER_ESP:
10937	case MSR_K8_FS_BASE:
10938	case MSR_K8_GS_BASE:
10939	{
10940	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10941	HMVMX_RETURN_UNEXPECTED_EXIT();
10942	}
10943
10944	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10945	default:
10946	{
10947	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10948	{
10949	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10950	if (pMixedCtx->ecx != MSR_K6_EFER)
10951	{
10952	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10953	pMixedCtx->ecx));
10954	HMVMX_RETURN_UNEXPECTED_EXIT();
10955	}
10956	}
10957
10958	#if HC_ARCH_BITS == 64
10959	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10960	{
10961	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10962	HMVMX_RETURN_UNEXPECTED_EXIT();
10963	}
10964	#endif
10965	break;
10966	}
10967	}
10968	}
10969	#endif /* VBOX_STRICT */
10970	}
10971	return rc;
10972	}
10973
10974
10975	/**
10976	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10977	*/
10978	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10979	{
10980	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10981
10982	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10983	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10984	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10985	return VERR_EM_INTERPRETER;
10986	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10987	HMVMX_RETURN_UNEXPECTED_EXIT();
10988	}
10989
10990
10991	/**
10992	* VM-exit handler for when the TPR value is lowered below the specified
10993	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10994	*/
10995	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10996	{
10997	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10998	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10999
11000	/*
11001	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
11002	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
11003	* resume guest execution.
11004	*/
11005	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
11006	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11007	return VINF_SUCCESS;
11008	}
11009
11010
11011	/**
11012	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11013	* VM-exit.
11014	*
11015	* @retval VINF_SUCCESS when guest execution can continue.
11016	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11017	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11018	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11019	* interpreter.
11020	*/
11021	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11022	{
11023	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11024	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11025	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11026	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11027	AssertRCReturn(rc, rc);
11028
11029	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11030	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
11031	PVM pVM = pVCpu->CTX_SUFF(pVM);
11032	VBOXSTRICTRC rcStrict;
11033	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
11034	switch (uAccessType)
11035	{
11036	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
11037	{
11038	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11039	AssertRCReturn(rc, rc);
11040
11041	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11042	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
11043	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
11044	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
11045	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11046	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
11047	{
11048	case 0: /* CR0 */
11049	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11050	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
11051	break;
11052	case 2: /* CR2 */
11053	/* Nothing to do here, CR2 it's not part of the VMCS. */
11054	break;
11055	case 3: /* CR3 */
11056	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
11057	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
11058	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
11059	break;
11060	case 4: /* CR4 */
11061	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
11062	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
11063	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11064	break;
11065	case 8: /* CR8 */
11066	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11067	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
11068	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
11069	break;
11070	default:
11071	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
11072	break;
11073	}
11074
11075	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
11076	break;
11077	}
11078
11079	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
11080	{
11081	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11082	AssertRCReturn(rc, rc);
11083
11084	Assert( !pVM->hm.s.fNestedPaging
11085	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
11086	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
11087
11088	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
11089	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
11090	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11091
11092	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
11093	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
11094	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
11095	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11096	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
11097	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
11098	VBOXSTRICTRC_VAL(rcStrict)));
11099	break;
11100	}
11101
11102	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
11103	{
11104	AssertRCReturn(rc, rc);
11105	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
11106	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11107	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11108	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
11109	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
11110	break;
11111	}
11112
11113	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
11114	{
11115	AssertRCReturn(rc, rc);
11116	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
11117	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
11118	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11119	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
11120	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
11121	break;
11122	}
11123
11124	default:
11125	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
11126	VERR_VMX_UNEXPECTED_EXCEPTION);
11127	}
11128
11129	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11130	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
11131	return VBOXSTRICTRC_TODO(rcStrict);
11132	}
11133
11134
11135	/**
11136	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
11137	* VM-exit.
11138	*/
11139	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11140	{
11141	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11142	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
11143
11144	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11145	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11146	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11147	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
11148	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
11149	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
11150	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
11151	AssertRCReturn(rc2, rc2);
11152
11153	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
11154	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
11155	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
11156	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
11157	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
11158	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
11159	bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11160	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
11161
11162	/* I/O operation lookup arrays. */
11163	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
11164	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
11165
11166	VBOXSTRICTRC rcStrict;
11167	uint32_t const cbValue = s_aIOSizes[uIOWidth];
11168	uint32_t const cbInstr = pVmxTransient->cbInstr;
11169	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
11170	PVM pVM = pVCpu->CTX_SUFF(pVM);
11171	if (fIOString)
11172	{
11173	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
11174	/*
11175	* INS/OUTS - I/O String instruction.
11176	*
11177	* Use instruction-information if available, otherwise fall back on
11178	* interpreting the instruction.
11179	*/
11180	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11181	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
11182	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
11183	{
11184	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11185	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11186	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11187	AssertRCReturn(rc2, rc2);
11188	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
11189	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
11190	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
11191	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
11192	if (fIOWrite)
11193	{
11194	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
11195	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
11196	}
11197	else
11198	{
11199	/*
11200	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
11201	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
11202	* See Intel Instruction spec. for "INS".
11203	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
11204	*/
11205	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
11206	}
11207	}
11208	else
11209	{
11210	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11211	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11212	AssertRCReturn(rc2, rc2);
11213	rcStrict = IEMExecOne(pVCpu);
11214	}
11215	/** @todo IEM needs to be setting these flags somehow. */
11216	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11217	fUpdateRipAlready = true;
11218	#else
11219	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11220	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
11221	if (RT_SUCCESS(rcStrict))
11222	{
11223	if (fIOWrite)
11224	{
11225	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11226	(DISCPUMODE)pDis->uAddrMode, cbValue);
11227	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
11228	}
11229	else
11230	{
11231	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11232	(DISCPUMODE)pDis->uAddrMode, cbValue);
11233	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
11234	}
11235	}
11236	else
11237	{
11238	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
11239	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11240	}
11241	#endif
11242	}
11243	else
11244	{
11245	/*
11246	* IN/OUT - I/O instruction.
11247	*/
11248	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11249	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11250	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11251	if (fIOWrite)
11252	{
11253	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11254	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11255	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11256	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11257	}
11258	else
11259	{
11260	uint32_t u32Result = 0;
11261	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11262	if (IOM_SUCCESS(rcStrict))
11263	{
11264	/* Save result of I/O IN instr. in AL/AX/EAX. */
11265	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
11266	}
11267	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11268	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11269	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11270	}
11271	}
11272
11273	if (IOM_SUCCESS(rcStrict))
11274	{
11275	if (!fUpdateRipAlready)
11276	{
11277	pMixedCtx->rip += cbInstr;
11278	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11279	}
11280
11281	/*
11282	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11283	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11284	*/
11285	if (fIOString)
11286	{
11287	/** @todo Single-step for INS/OUTS with REP prefix? */
11288	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11289	}
11290	else if (fStepping)
11291	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11292
11293	/*
11294	* If any I/O breakpoints are armed, we need to check if one triggered
11295	* and take appropriate action.
11296	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11297	*/
11298	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11299	AssertRCReturn(rc2, rc2);
11300
11301	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11302	* execution engines about whether hyper BPs and such are pending. */
11303	uint32_t const uDr7 = pMixedCtx->dr[7];
11304	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11305	&& X86_DR7_ANY_RW_IO(uDr7)
11306	&& (pMixedCtx->cr4 & X86_CR4_DE))
11307	\|\| DBGFBpIsHwIoArmed(pVM)))
11308	{
11309	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11310
11311	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11312	VMMRZCallRing3Disable(pVCpu);
11313	HM_DISABLE_PREEMPT();
11314
11315	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11316
11317	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11318	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11319	{
11320	/* Raise #DB. */
11321	if (fIsGuestDbgActive)
11322	ASMSetDR6(pMixedCtx->dr[6]);
11323	if (pMixedCtx->dr[7] != uDr7)
11324	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11325
11326	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11327	}
11328	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11329	else if ( rcStrict2 != VINF_SUCCESS
11330	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
11331	rcStrict = rcStrict2;
11332
11333	HM_RESTORE_PREEMPT();
11334	VMMRZCallRing3Enable(pVCpu);
11335	}
11336	}
11337
11338	#ifdef DEBUG
11339	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11340	Assert(!fIOWrite);
11341	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11342	Assert(fIOWrite);
11343	else
11344	{
11345	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11346	* statuses, that the VMM device and some others may return. See
11347	* IOM_SUCCESS() for guidance. */
11348	AssertMsg( RT_FAILURE(rcStrict)
11349	\|\| rcStrict == VINF_SUCCESS
11350	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
11351	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
11352	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
11353	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11354	}
11355	#endif
11356
11357	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11358	return VBOXSTRICTRC_TODO(rcStrict);
11359	}
11360
11361
11362	/**
11363	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11364	* VM-exit.
11365	*/
11366	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11367	{
11368	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11369
11370	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
11371	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11372	AssertRCReturn(rc, rc);
11373	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11374	{
11375	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11376	AssertRCReturn(rc, rc);
11377	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11378	{
11379	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11380
11381	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11382	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11383
11384	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11385	Assert(!pVCpu->hm.s.Event.fPending);
11386	pVCpu->hm.s.Event.fPending = true;
11387	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11388	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11389	AssertRCReturn(rc, rc);
11390	if (fErrorCodeValid)
11391	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11392	else
11393	pVCpu->hm.s.Event.u32ErrCode = 0;
11394	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11395	&& uVector == X86_XCPT_PF)
11396	{
11397	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11398	}
11399
11400	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11401	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11402	return VINF_EM_RAW_INJECT_TRPM_EVENT;
11403	}
11404	}
11405
11406	/** @todo Emulate task switch someday, currently just going back to ring-3 for
11407	* emulation. */
11408	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11409	return VERR_EM_INTERPRETER;
11410	}
11411
11412
11413	/**
11414	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11415	*/
11416	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11417	{
11418	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11419	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11420	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11421	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11422	AssertRCReturn(rc, rc);
11423	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11424	return VINF_EM_DBG_STEPPED;
11425	}
11426
11427
11428	/**
11429	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11430	*/
11431	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11432	{
11433	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11434
11435	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11436	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11437	if (RT_UNLIKELY(rc != VINF_SUCCESS))
11438	{
11439	if (rc == VINF_HM_DOUBLE_FAULT)
11440	rc = VINF_SUCCESS;
11441	return rc;
11442	}
11443
11444	#if 0
11445	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11446	* just sync the whole thing. */
11447	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11448	#else
11449	/* Aggressive state sync. for now. */
11450	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11451	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11452	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11453	#endif
11454	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11455	AssertRCReturn(rc, rc);
11456
11457	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11458	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11459	switch (uAccessType)
11460	{
11461	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11462	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11463	{
11464	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11465	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11466	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11467
11468	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11469	GCPhys &= PAGE_BASE_GC_MASK;
11470	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11471	PVM pVM = pVCpu->CTX_SUFF(pVM);
11472	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11473	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11474
11475	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11476	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11477	CPUMCTX2CORE(pMixedCtx), GCPhys);
11478	rc = VBOXSTRICTRC_VAL(rc2);
11479	Log4(("ApicAccess rc=%d\n", rc));
11480	if ( rc == VINF_SUCCESS
11481	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
11482	\|\| rc == VERR_PAGE_NOT_PRESENT)
11483	{
11484	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11485	\| HM_CHANGED_GUEST_RSP
11486	\| HM_CHANGED_GUEST_RFLAGS
11487	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11488	rc = VINF_SUCCESS;
11489	}
11490	break;
11491	}
11492
11493	default:
11494	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11495	rc = VINF_EM_RAW_EMULATE_INSTR;
11496	break;
11497	}
11498
11499	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11500	if (rc != VINF_SUCCESS)
11501	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
11502	return rc;
11503	}
11504
11505
11506	/**
11507	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11508	* VM-exit.
11509	*/
11510	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11511	{
11512	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11513
11514	/* We should -not- get this VM-exit if the guest's debug registers were active. */
11515	if (pVmxTransient->fWasGuestDebugStateActive)
11516	{
11517	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11518	HMVMX_RETURN_UNEXPECTED_EXIT();
11519	}
11520
11521	int rc = VERR_INTERNAL_ERROR_5;
11522	if ( !DBGFIsStepping(pVCpu)
11523	&& !pVCpu->hm.s.fSingleInstruction
11524	&& !pVmxTransient->fWasHyperDebugStateActive)
11525	{
11526	/* Don't intercept MOV DRx and #DB any more. */
11527	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11528	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11529	AssertRCReturn(rc, rc);
11530
11531	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11532	{
11533	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11534	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11535	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
11536	#endif
11537	}
11538
11539	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11540	VMMRZCallRing3Disable(pVCpu);
11541	HM_DISABLE_PREEMPT();
11542
11543	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11544	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11545	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
11546
11547	HM_RESTORE_PREEMPT();
11548	VMMRZCallRing3Enable(pVCpu);
11549
11550	#ifdef VBOX_WITH_STATISTICS
11551	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11552	AssertRCReturn(rc, rc);
11553	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11554	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11555	else
11556	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11557	#endif
11558	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11559	return VINF_SUCCESS;
11560	}
11561
11562	/*
11563	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11564	* Update the segment registers and DR7 from the CPU.
11565	*/
11566	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11567	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11568	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11569	AssertRCReturn(rc, rc);
11570	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11571
11572	PVM pVM = pVCpu->CTX_SUFF(pVM);
11573	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11574	{
11575	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11576	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11577	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11578	if (RT_SUCCESS(rc))
11579	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11580	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11581	}
11582	else
11583	{
11584	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11585	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11586	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11587	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11588	}
11589
11590	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
11591	if (RT_SUCCESS(rc))
11592	{
11593	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11594	AssertRCReturn(rc2, rc2);
11595	}
11596	return rc;
11597	}
11598
11599
11600	/**
11601	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11602	* Conditional VM-exit.
11603	*/
11604	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11605	{
11606	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11607	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11608
11609	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11610	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11611	if (RT_UNLIKELY(rc != VINF_SUCCESS))
11612	{
11613	if (rc == VINF_HM_DOUBLE_FAULT)
11614	rc = VINF_SUCCESS;
11615	return rc;
11616	}
11617
11618	RTGCPHYS GCPhys = 0;
11619	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11620
11621	#if 0
11622	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11623	#else
11624	/* Aggressive state sync. for now. */
11625	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11626	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11627	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11628	#endif
11629	AssertRCReturn(rc, rc);
11630
11631	/*
11632	* If we succeed, resume guest execution.
11633	* If we fail in interpreting the instruction because we couldn't get the guest physical address
11634	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11635	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11636	* weird case. See @bugref{6043}.
11637	*/
11638	PVM pVM = pVCpu->CTX_SUFF(pVM);
11639	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11640	rc = VBOXSTRICTRC_VAL(rc2);
11641	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11642	if ( rc == VINF_SUCCESS
11643	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
11644	\|\| rc == VERR_PAGE_NOT_PRESENT)
11645	{
11646	/* Successfully handled MMIO operation. */
11647	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11648	\| HM_CHANGED_GUEST_RSP
11649	\| HM_CHANGED_GUEST_RFLAGS
11650	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11651	rc = VINF_SUCCESS;
11652	}
11653	return rc;
11654	}
11655
11656
11657	/**
11658	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11659	* VM-exit.
11660	*/
11661	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11662	{
11663	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11664	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11665
11666	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11667	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11668	if (RT_UNLIKELY(rc != VINF_SUCCESS))
11669	{
11670	if (rc == VINF_HM_DOUBLE_FAULT)
11671	rc = VINF_SUCCESS;
11672	return rc;
11673	}
11674
11675	RTGCPHYS GCPhys = 0;
11676	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11677	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11678	#if 0
11679	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11680	#else
11681	/* Aggressive state sync. for now. */
11682	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11683	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11684	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11685	#endif
11686	AssertRCReturn(rc, rc);
11687
11688	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11689	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11690
11691	RTGCUINT uErrorCode = 0;
11692	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11693	uErrorCode \|= X86_TRAP_PF_ID;
11694	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11695	uErrorCode \|= X86_TRAP_PF_RW;
11696	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11697	uErrorCode \|= X86_TRAP_PF_P;
11698
11699	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11700
11701	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11702	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11703
11704	/* Handle the pagefault trap for the nested shadow table. */
11705	PVM pVM = pVCpu->CTX_SUFF(pVM);
11706	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11707	TRPMResetTrap(pVCpu);
11708
11709	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11710	if ( rc == VINF_SUCCESS
11711	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
11712	\|\| rc == VERR_PAGE_NOT_PRESENT)
11713	{
11714	/* Successfully synced our nested page tables. */
11715	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11716	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11717	\| HM_CHANGED_GUEST_RSP
11718	\| HM_CHANGED_GUEST_RFLAGS);
11719	return VINF_SUCCESS;
11720	}
11721
11722	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11723	return rc;
11724	}
11725
11726	/** @} */
11727
11728	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11729	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11730	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11731
11732	/** @name VM-exit exception handlers.
11733	* @{
11734	*/
11735
11736	/**
11737	* VM-exit exception handler for #MF (Math Fault: floating point exception).
11738	*/
11739	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11740	{
11741	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11742	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11743
11744	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11745	AssertRCReturn(rc, rc);
11746
11747	if (!(pMixedCtx->cr0 & X86_CR0_NE))
11748	{
11749	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11750	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11751
11752	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11753	* provides VM-exit instruction length. If this causes problem later,
11754	* disassemble the instruction like it's done on AMD-V. */
11755	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11756	AssertRCReturn(rc2, rc2);
11757	return rc;
11758	}
11759
11760	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11761	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11762	return rc;
11763	}
11764
11765
11766	/**
11767	* VM-exit exception handler for #BP (Breakpoint exception).
11768	*/
11769	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11770	{
11771	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11772	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11773
11774	/** @todo Try optimize this by not saving the entire guest state unless
11775	* really needed. */
11776	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11777	AssertRCReturn(rc, rc);
11778
11779	PVM pVM = pVCpu->CTX_SUFF(pVM);
11780	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11781	if (rc == VINF_EM_RAW_GUEST_TRAP)
11782	{
11783	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11784	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11785	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11786	AssertRCReturn(rc, rc);
11787
11788	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11789	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11790	}
11791
11792	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
11793	return rc;
11794	}
11795
11796
11797	/**
11798	* VM-exit exception handler for #DB (Debug exception).
11799	*/
11800	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11801	{
11802	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11803	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11804	Log6(("XcptDB\n"));
11805
11806	/*
11807	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11808	* for processing.
11809	*/
11810	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11811	AssertRCReturn(rc, rc);
11812
11813	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11814	uint64_t uDR6 = X86_DR6_INIT_VAL;
11815	uDR6 \|= ( pVmxTransient->uExitQualification
11816	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
11817
11818	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11819	if (rc == VINF_EM_RAW_GUEST_TRAP)
11820	{
11821	/*
11822	* The exception was for the guest. Update DR6, DR7.GD and
11823	* IA32_DEBUGCTL.LBR before forwarding it.
11824	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11825	*/
11826	VMMRZCallRing3Disable(pVCpu);
11827	HM_DISABLE_PREEMPT();
11828
11829	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11830	pMixedCtx->dr[6] \|= uDR6;
11831	if (CPUMIsGuestDebugStateActive(pVCpu))
11832	ASMSetDR6(pMixedCtx->dr[6]);
11833
11834	HM_RESTORE_PREEMPT();
11835	VMMRZCallRing3Enable(pVCpu);
11836
11837	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11838	AssertRCReturn(rc, rc);
11839
11840	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11841	pMixedCtx->dr[7] &= ~X86_DR7_GD;
11842
11843	/* Paranoia. */
11844	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11845	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
11846
11847	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11848	AssertRCReturn(rc, rc);
11849
11850	/*
11851	* Raise #DB in the guest.
11852	*
11853	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11854	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11855	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11856	*
11857	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11858	*/
11859	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11860	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11861	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11862	AssertRCReturn(rc, rc);
11863	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11864	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11865	return VINF_SUCCESS;
11866	}
11867
11868	/*
11869	* Not a guest trap, must be a hypervisor related debug event then.
11870	* Update DR6 in case someone is interested in it.
11871	*/
11872	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11873	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11874	CPUMSetHyperDR6(pVCpu, uDR6);
11875
11876	return rc;
11877	}
11878
11879
11880	/**
11881	* VM-exit exception handler for #NM (Device-not-available exception: floating
11882	* point exception).
11883	*/
11884	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11885	{
11886	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11887
11888	/* We require CR0 and EFER. EFER is always up-to-date. */
11889	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11890	AssertRCReturn(rc, rc);
11891
11892	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11893	VMMRZCallRing3Disable(pVCpu);
11894	HM_DISABLE_PREEMPT();
11895
11896	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11897	if (pVmxTransient->fWasGuestFPUStateActive)
11898	{
11899	rc = VINF_EM_RAW_GUEST_TRAP;
11900	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11901	}
11902	else
11903	{
11904	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11905	Assert(!pVmxTransient->fWasGuestFPUStateActive);
11906	#endif
11907	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11908	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11909	}
11910
11911	HM_RESTORE_PREEMPT();
11912	VMMRZCallRing3Enable(pVCpu);
11913
11914	if (rc == VINF_SUCCESS)
11915	{
11916	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11917	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11918	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11919	pVCpu->hm.s.fPreloadGuestFpu = true;
11920	}
11921	else
11922	{
11923	/* Forward #NM to the guest. */
11924	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11925	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11926	AssertRCReturn(rc, rc);
11927	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11928	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11929	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11930	}
11931
11932	return VINF_SUCCESS;
11933	}
11934
11935
11936	/**
11937	* VM-exit exception handler for #GP (General-protection exception).
11938	*
11939	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11940	*/
11941	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11942	{
11943	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11944	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11945
11946	int rc = VERR_INTERNAL_ERROR_5;
11947	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11948	{
11949	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11950	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11951	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11952	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11953	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11954	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11955	AssertRCReturn(rc, rc);
11956	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11957	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11958	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11959	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11960	return rc;
11961	#else
11962	/* We don't intercept #GP. */
11963	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11964	NOREF(pVmxTransient);
11965	return VERR_VMX_UNEXPECTED_EXCEPTION;
11966	#endif
11967	}
11968
11969	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11970	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11971
11972	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11973	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11974	AssertRCReturn(rc, rc);
11975
11976	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11977	uint32_t cbOp = 0;
11978	PVM pVM = pVCpu->CTX_SUFF(pVM);
11979	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11980	if (RT_SUCCESS(rc))
11981	{
11982	rc = VINF_SUCCESS;
11983	Assert(cbOp == pDis->cbInstr);
11984	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11985	switch (pDis->pCurInstr->uOpcode)
11986	{
11987	case OP_CLI:
11988	{
11989	pMixedCtx->eflags.Bits.u1IF = 0;
11990	pMixedCtx->eflags.Bits.u1RF = 0;
11991	pMixedCtx->rip += pDis->cbInstr;
11992	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11993	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11994	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11995	break;
11996	}
11997
11998	case OP_STI:
11999	{
12000	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
12001	pMixedCtx->eflags.Bits.u1IF = 1;
12002	pMixedCtx->eflags.Bits.u1RF = 0;
12003	pMixedCtx->rip += pDis->cbInstr;
12004	if (!fOldIF)
12005	{
12006	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
12007	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12008	}
12009	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
12010	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
12011	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12012	break;
12013	}
12014
12015	case OP_HLT:
12016	{
12017	rc = VINF_EM_HALT;
12018	pMixedCtx->rip += pDis->cbInstr;
12019	pMixedCtx->eflags.Bits.u1RF = 0;
12020	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
12021	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12022	break;
12023	}
12024
12025	case OP_POPF:
12026	{
12027	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12028	uint32_t cbParm;
12029	uint32_t uMask;
12030	bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12031	if (pDis->fPrefix & DISPREFIX_OPSIZE)
12032	{
12033	cbParm = 4;
12034	uMask = 0xffffffff;
12035	}
12036	else
12037	{
12038	cbParm = 2;
12039	uMask = 0xffff;
12040	}
12041
12042	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
12043	RTGCPTR GCPtrStack = 0;
12044	X86EFLAGS Eflags;
12045	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12046	&GCPtrStack);
12047	if (RT_SUCCESS(rc))
12048	{
12049	Assert(sizeof(Eflags.u32) >= cbParm);
12050	Eflags.u32 = 0;
12051	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
12052	}
12053	if (RT_FAILURE(rc))
12054	{
12055	rc = VERR_EM_INTERPRETER;
12056	break;
12057	}
12058	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
12059	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
12060	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
12061	pMixedCtx->esp += cbParm;
12062	pMixedCtx->esp &= uMask;
12063	pMixedCtx->rip += pDis->cbInstr;
12064	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12065	\| HM_CHANGED_GUEST_RSP
12066	\| HM_CHANGED_GUEST_RFLAGS);
12067	/* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
12068	if (fStepping)
12069	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
12070
12071	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
12072	break;
12073	}
12074
12075	case OP_PUSHF:
12076	{
12077	uint32_t cbParm;
12078	uint32_t uMask;
12079	if (pDis->fPrefix & DISPREFIX_OPSIZE)
12080	{
12081	cbParm = 4;
12082	uMask = 0xffffffff;
12083	}
12084	else
12085	{
12086	cbParm = 2;
12087	uMask = 0xffff;
12088	}
12089
12090	/* Get the stack pointer & push the contents of eflags onto the stack. */
12091	RTGCPTR GCPtrStack = 0;
12092	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
12093	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
12094	if (RT_FAILURE(rc))
12095	{
12096	rc = VERR_EM_INTERPRETER;
12097	break;
12098	}
12099	X86EFLAGS Eflags = pMixedCtx->eflags;
12100	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
12101	Eflags.Bits.u1RF = 0;
12102	Eflags.Bits.u1VM = 0;
12103
12104	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
12105	if (RT_FAILURE(rc))
12106	{
12107	rc = VERR_EM_INTERPRETER;
12108	break;
12109	}
12110	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
12111	pMixedCtx->esp -= cbParm;
12112	pMixedCtx->esp &= uMask;
12113	pMixedCtx->rip += pDis->cbInstr;
12114	pMixedCtx->eflags.Bits.u1RF = 0;
12115	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12116	\| HM_CHANGED_GUEST_RSP
12117	\| HM_CHANGED_GUEST_RFLAGS);
12118	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
12119	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
12120	break;
12121	}
12122
12123	case OP_IRET:
12124	{
12125	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
12126	* instruction reference. */
12127	RTGCPTR GCPtrStack = 0;
12128	uint32_t uMask = 0xffff;
12129	bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12130	uint16_t aIretFrame[3];
12131	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
12132	{
12133	rc = VERR_EM_INTERPRETER;
12134	break;
12135	}
12136	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12137	&GCPtrStack);
12138	if (RT_SUCCESS(rc))
12139	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
12140	if (RT_FAILURE(rc))
12141	{
12142	rc = VERR_EM_INTERPRETER;
12143	break;
12144	}
12145	pMixedCtx->eip = 0;
12146	pMixedCtx->ip = aIretFrame[0];
12147	pMixedCtx->cs.Sel = aIretFrame[1];
12148	pMixedCtx->cs.ValidSel = aIretFrame[1];
12149	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
12150	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
12151	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
12152	pMixedCtx->sp += sizeof(aIretFrame);
12153	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12154	\| HM_CHANGED_GUEST_SEGMENT_REGS
12155	\| HM_CHANGED_GUEST_RSP
12156	\| HM_CHANGED_GUEST_RFLAGS);
12157	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
12158	if (fStepping)
12159	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
12160	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
12161	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
12162	break;
12163	}
12164
12165	case OP_INT:
12166	{
12167	uint16_t uVector = pDis->Param1.uValue & 0xff;
12168	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
12169	/* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12170	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12171	break;
12172	}
12173
12174	case OP_INTO:
12175	{
12176	if (pMixedCtx->eflags.Bits.u1OF)
12177	{
12178	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
12179	/* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12180	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12181	}
12182	else
12183	{
12184	pMixedCtx->eflags.Bits.u1RF = 0;
12185	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12186	}
12187	break;
12188	}
12189
12190	default:
12191	{
12192	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
12193	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
12194	EMCODETYPE_SUPERVISOR);
12195	rc = VBOXSTRICTRC_VAL(rc2);
12196	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
12197	/** @todo We have to set pending-debug exceptions here when the guest is
12198	* single-stepping depending on the instruction that was interpreted. */
12199	Log4(("#GP rc=%Rrc\n", rc));
12200	break;
12201	}
12202	}
12203	}
12204	else
12205	rc = VERR_EM_INTERPRETER;
12206
12207	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
12208	("#GP Unexpected rc=%Rrc\n", rc));
12209	return rc;
12210	}
12211
12212
12213	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12214	/**
12215	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
12216	* the exception reported in the VMX transient structure back into the VM.
12217	*
12218	* @remarks Requires uExitIntInfo in the VMX transient structure to be
12219	* up-to-date.
12220	*/
12221	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12222	{
12223	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12224
12225	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
12226	hmR0VmxCheckExitDueToEventDelivery(). */
12227	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12228	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12229	AssertRCReturn(rc, rc);
12230	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12231
12232	#ifdef DEBUG_ramshankar
12233	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12234	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12235	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12236	#endif
12237
12238	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12239	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12240	return VINF_SUCCESS;
12241	}
12242	#endif
12243
12244
12245	/**
12246	* VM-exit exception handler for #PF (Page-fault exception).
12247	*/
12248	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12249	{
12250	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12251	PVM pVM = pVCpu->CTX_SUFF(pVM);
12252	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12253	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12254	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12255	AssertRCReturn(rc, rc);
12256
12257	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
12258	if (pVM->hm.s.fNestedPaging)
12259	{
12260	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12261	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12262	{
12263	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12264	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12265	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12266	}
12267	else
12268	{
12269	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12270	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12271	Log4(("Pending #DF due to vectoring #PF. NP\n"));
12272	}
12273	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12274	return rc;
12275	}
12276	#else
12277	Assert(!pVM->hm.s.fNestedPaging);
12278	NOREF(pVM);
12279	#endif
12280
12281	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12282	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12283	if (pVmxTransient->fVectoringPF)
12284	{
12285	Assert(pVCpu->hm.s.Event.fPending);
12286	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12287	}
12288
12289	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12290	AssertRCReturn(rc, rc);
12291
12292	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12293	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12294
12295	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12296	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12297	(RTGCPTR)pVmxTransient->uExitQualification);
12298
12299	Log4(("#PF: rc=%Rrc\n", rc));
12300	if (rc == VINF_SUCCESS)
12301	{
12302	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12303	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
12304	* memory? We don't update the whole state here... */
12305	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12306	\| HM_CHANGED_GUEST_RSP
12307	\| HM_CHANGED_GUEST_RFLAGS
12308	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
12309	TRPMResetTrap(pVCpu);
12310	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12311	return rc;
12312	}
12313
12314	if (rc == VINF_EM_RAW_GUEST_TRAP)
12315	{
12316	if (!pVmxTransient->fVectoringDoublePF)
12317	{
12318	/* It's a guest page fault and needs to be reflected to the guest. */
12319	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12320	TRPMResetTrap(pVCpu);
12321	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12322	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12323	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12324	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12325	}
12326	else
12327	{
12328	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12329	TRPMResetTrap(pVCpu);
12330	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12331	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12332	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12333	}
12334
12335	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12336	return VINF_SUCCESS;
12337	}
12338
12339	TRPMResetTrap(pVCpu);
12340	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12341	return rc;
12342	}
12343
12344	/** @} */
12345

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 55810

Download in other formats: