HMVMXR0.cpp@ 49729

Last change on this file since 49729 was 49729, checked in by vboxsync, 11 years ago
VMM/HM: VMCPU_HMCF -> HMCPU_CF macro renaming.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 470.3 KB

Line
1	/* $Id: HMVMXR0.cpp 49729 2013-11-29 14:20:44Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/x86.h>
23	#include <iprt/asm-amd64-x86.h>
24	#include <iprt/thread.h>
25	#include <iprt/string.h>
26
27	#include "HMInternal.h"
28	#include <VBox/vmm/vm.h>
29	#include "HMVMXR0.h"
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#ifdef VBOX_WITH_REM
37	# include <VBox/vmm/rem.h>
38	#endif
39	#ifdef DEBUG_ramshankar
40	# define HMVMX_SAVE_FULL_GUEST_STATE
41	# define HMVMX_SYNC_FULL_GUEST_STATE
42	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44	# define HMVMX_ALWAYS_TRAP_PF
45	# define HMVMX_ALWAYS_SWAP_FPU_STATE
46	# define HMVMX_ALWAYS_FLUSH_TLB
47	#endif
48
49
50	/*******************************************************************************
51	* Defined Constants And Macros *
52	*******************************************************************************/
53	#if defined(RT_ARCH_AMD64)
54	# define HMVMX_IS_64BIT_HOST_MODE() (true)
55	typedef RTHCUINTREG HMVMXHCUINTREG;
56	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57	extern "C" uint32_t g_fVMXIs64bitHost;
58	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	typedef uint64_t HMVMXHCUINTREG;
60	#else
61	# define HMVMX_IS_64BIT_HOST_MODE() (false)
62	typedef RTHCUINTREG HMVMXHCUINTREG;
63	#endif
64
65	/** Use the function table. */
66	#define HMVMX_USE_FUNCTION_TABLE
67
68	/** Determine which tagged-TLB flush handler to use. */
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74	/** @name Updated-guest-state flags.
75	* @{ */
76	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96	\| HMVMX_UPDATED_GUEST_RSP \
97	\| HMVMX_UPDATED_GUEST_RFLAGS \
98	\| HMVMX_UPDATED_GUEST_CR0 \
99	\| HMVMX_UPDATED_GUEST_CR3 \
100	\| HMVMX_UPDATED_GUEST_CR4 \
101	\| HMVMX_UPDATED_GUEST_GDTR \
102	\| HMVMX_UPDATED_GUEST_IDTR \
103	\| HMVMX_UPDATED_GUEST_LDTR \
104	\| HMVMX_UPDATED_GUEST_TR \
105	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106	\| HMVMX_UPDATED_GUEST_DEBUG \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127	/** @} */
128
129	/** @name
130	* States of the VMCS.
131	*
132	* This does not reflect all possible VMCS states but currently only those
133	* needed for maintaining the VMCS consistently even when thread-context hooks
134	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
135	*/
136	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139	/** @} */
140
141	/**
142	* Exception bitmap mask for real-mode guests (real-on-v86).
143	*
144	* We need to intercept all exceptions manually (except #PF). #NM is also
145	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146	* even in real-mode if we have Nested Paging support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*******************************************************************************
197	* Structures and Typedefs *
198	*******************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG uEflags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u6Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringPF;
278	} VMXTRANSIENT;
279	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284	/** Pointer to VMX transient state. */
285	typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288	/**
289	* MSR-bitmap read permissions.
290	*/
291	typedef enum VMXMSREXITREAD
292	{
293	/** Reading this MSR causes a VM-exit. */
294	VMXMSREXIT_INTERCEPT_READ = 0xb,
295	/** Reading this MSR does not cause a VM-exit. */
296	VMXMSREXIT_PASSTHRU_READ
297	} VMXMSREXITREAD;
298	/** Pointer to MSR-bitmap read permissions. */
299	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311	/** Pointer to MSR-bitmap write permissions. */
312	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315	/**
316	* VMX VM-exit handler.
317	*
318	* @returns VBox status code.
319	* @param pVCpu Pointer to the VMCPU.
320	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
321	* out-of-sync. Make sure to update the required
322	* fields before using them.
323	* @param pVmxTransient Pointer to the VMX-transient structure.
324	*/
325	#ifndef HMVMX_USE_FUNCTION_TABLE
326	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327	#else
328	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	/** Pointer to VM-exit handler. */
330	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331	#endif
332
333
334	/*******************************************************************************
335	* Internal Functions *
336	*******************************************************************************/
337	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343	#endif
344	#ifndef HMVMX_USE_FUNCTION_TABLE
345	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346	# define HMVMX_EXIT_DECL static int
347	#else
348	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349	#endif
350
351	/** @name VM-exit handlers.
352	* @{
353	*/
354	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358	static FNVMXEXITHANDLER hmR0VmxExitSipi;
359	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360	static FNVMXEXITHANDLER hmR0VmxExitSmi;
361	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366	static FNVMXEXITHANDLER hmR0VmxExitHlt;
367	static FNVMXEXITHANDLER hmR0VmxExitInvd;
368	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371	static FNVMXEXITHANDLER hmR0VmxExitRsm;
372	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381	static FNVMXEXITHANDLER hmR0VmxExitMwait;
382	static FNVMXEXITHANDLER hmR0VmxExitMtf;
383	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384	static FNVMXEXITHANDLER hmR0VmxExitPause;
385	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398	/** @} */
399
400	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408	#endif
409	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411	/*******************************************************************************
412	* Global Variables *
413	*******************************************************************************/
414	#ifdef HMVMX_USE_FUNCTION_TABLE
415
416	/**
417	* VMX_EXIT dispatch table.
418	*/
419	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420	{
421	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461	/* 40 UNDEFINED */ hmR0VmxExitPause,
462	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481	};
482	#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484	#ifdef VBOX_STRICT
485	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486	{
487	/* 0 */ "(Not Used)",
488	/* 1 */ "VMCALL executed in VMX root operation.",
489	/* 2 */ "VMCLEAR with invalid physical address.",
490	/* 3 */ "VMCLEAR with VMXON pointer.",
491	/* 4 */ "VMLAUNCH with non-clear VMCS.",
492	/* 5 */ "VMRESUME with non-launched VMCS.",
493	/* 6 */ "VMRESUME after VMXOFF",
494	/* 7 */ "VM entry with invalid control fields.",
495	/* 8 */ "VM entry with invalid host state fields.",
496	/* 9 */ "VMPTRLD with invalid physical address.",
497	/* 10 */ "VMPTRLD with VMXON pointer.",
498	/* 11 */ "VMPTRLD with incorrect revision identifier.",
499	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500	/* 13 */ "VMWRITE to read-only VMCS component.",
501	/* 14 */ "(Not Used)",
502	/* 15 */ "VMXON executed in VMX root operation.",
503	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
504	/* 17 */ "VM entry with non-launched executing VMCS.",
505	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506	/* 19 */ "VMCALL with non-clear VMCS.",
507	/* 20 */ "VMCALL with invalid VM-exit control fields.",
508	/* 21 */ "(Not Used)",
509	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511	/* 24 */ "VMCALL with invalid SMM-monitor features.",
512	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513	/* 26 */ "VM entry with events blocked by MOV SS.",
514	/* 27 */ "(Not Used)",
515	/* 28 */ "Invalid operand to INVEPT/INVVPID."
516	};
517	#endif /* VBOX_STRICT */
518
519
520
521	/**
522	* Updates the VM's last error record. If there was a VMX instruction error,
523	* reads the error data from the VMCS and updates VCPU's last error record as
524	* well.
525	*
526	* @param pVM Pointer to the VM.
527	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528	* VERR_VMX_UNABLE_TO_START_VM or
529	* VERR_VMX_INVALID_VMCS_FIELD).
530	* @param rc The error code.
531	*/
532	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533	{
534	AssertPtr(pVM);
535	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
537	{
538	AssertPtrReturnVoid(pVCpu);
539	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540	}
541	pVM->hm.s.lLastError = rc;
542	}
543
544
545	/**
546	* Reads the VM-entry interruption-information field from the VMCS into the VMX
547	* transient structure.
548	*
549	* @returns VBox status code.
550	* @param pVmxTransient Pointer to the VMX transient structure.
551	*
552	* @remarks No-long-jump zone!!!
553	*/
554	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555	{
556	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557	AssertRCReturn(rc, rc);
558	return VINF_SUCCESS;
559	}
560
561
562	/**
563	* Reads the VM-entry exception error code field from the VMCS into
564	* the VMX transient structure.
565	*
566	* @returns VBox status code.
567	* @param pVmxTransient Pointer to the VMX transient structure.
568	*
569	* @remarks No-long-jump zone!!!
570	*/
571	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572	{
573	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574	AssertRCReturn(rc, rc);
575	return VINF_SUCCESS;
576	}
577
578
579	/**
580	* Reads the VM-entry exception error code field from the VMCS into
581	* the VMX transient structure.
582	*
583	* @returns VBox status code.
584	* @param pVmxTransient Pointer to the VMX transient structure.
585	*
586	* @remarks No-long-jump zone!!!
587	*/
588	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589	{
590	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591	AssertRCReturn(rc, rc);
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the VM-exit interruption-information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*/
603	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the VM-exit interruption error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Reads the VM-exit instruction length field from the VMCS into the VMX
636	* transient structure.
637	*
638	* @returns VBox status code.
639	* @param pVCpu Pointer to the VMCPU.
640	* @param pVmxTransient Pointer to the VMX transient structure.
641	*/
642	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643	{
644	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645	{
646	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647	AssertRCReturn(rc, rc);
648	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649	}
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Reads the VM-exit instruction-information field from the VMCS into
656	* the VMX transient structure.
657	*
658	* @returns VBox status code.
659	* @param pVmxTransient Pointer to the VMX transient structure.
660	*/
661	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662	{
663	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664	{
665	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666	AssertRCReturn(rc, rc);
667	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668	}
669	return VINF_SUCCESS;
670	}
671
672
673	/**
674	* Reads the exit qualification from the VMCS into the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678	* case).
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684	{
685	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the IDT-vectoring information field from the VMCS into the VMX
695	* transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*
700	* @remarks No-long-jump zone!!!
701	*/
702	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703	{
704	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705	{
706	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707	AssertRCReturn(rc, rc);
708	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709	}
710	return VINF_SUCCESS;
711	}
712
713
714	/**
715	* Reads the IDT-vectoring error code from the VMCS into the VMX
716	* transient structure.
717	*
718	* @returns VBox status code.
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724	{
725	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Enters VMX root mode operation on the current CPU.
735	*
736	* @returns VBox status code.
737	* @param pVM Pointer to the VM (optional, can be NULL, after
738	* a resume).
739	* @param HCPhysCpuPage Physical address of the VMXON region.
740	* @param pvCpuPage Pointer to the VMXON region.
741	*/
742	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743	{
744	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
745	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
746	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
747
748	if (pVM)
749	{
750	/* Write the VMCS revision dword to the VMXON region. */
751	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
752	}
753
754	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
755	RTCCUINTREG uEflags = ASMIntDisableFlags();
756
757	/* Enable the VMX bit in CR4 if necessary. */
758	RTCCUINTREG uCr4 = ASMGetCR4();
759	if (!(uCr4 & X86_CR4_VMXE))
760	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
761
762	/* Enter VMX root mode. */
763	int rc = VMXEnable(HCPhysCpuPage);
764	if (RT_FAILURE(rc))
765	ASMSetCR4(uCr4);
766
767	/* Restore interrupts. */
768	ASMSetFlags(uEflags);
769	return rc;
770	}
771
772
773	/**
774	* Exits VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	*/
778	static int hmR0VmxLeaveRootMode(void)
779	{
780	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
781
782	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
783	RTCCUINTREG uEflags = ASMIntDisableFlags();
784
785	/* If we're for some reason not in VMX root mode, then don't leave it. */
786	RTCCUINTREG uHostCR4 = ASMGetCR4();
787
788	int rc;
789	if (uHostCR4 & X86_CR4_VMXE)
790	{
791	/* Exit VMX root mode and clear the VMX bit in CR4. */
792	VMXDisable();
793	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
794	rc = VINF_SUCCESS;
795	}
796	else
797	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
798
799	/* Restore interrupts. */
800	ASMSetFlags(uEflags);
801	return rc;
802	}
803
804
805	/**
806	* Allocates and maps one physically contiguous page. The allocated page is
807	* zero'd out. (Used by various VT-x structures).
808	*
809	* @returns IPRT status code.
810	* @param pMemObj Pointer to the ring-0 memory object.
811	* @param ppVirt Where to store the virtual address of the
812	* allocation.
813	* @param pPhys Where to store the physical address of the
814	* allocation.
815	*/
816	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
817	{
818	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
819	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
820	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
821
822	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
823	if (RT_FAILURE(rc))
824	return rc;
825	ppVirt = RTR0MemObjAddress(pMemObj);
826	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
827	ASMMemZero32(*ppVirt, PAGE_SIZE);
828	return VINF_SUCCESS;
829	}
830
831
832	/**
833	* Frees and unmaps an allocated physical page.
834	*
835	* @param pMemObj Pointer to the ring-0 memory object.
836	* @param ppVirt Where to re-initialize the virtual address of
837	* allocation as 0.
838	* @param pHCPhys Where to re-initialize the physical address of the
839	* allocation as 0.
840	*/
841	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
842	{
843	AssertPtr(pMemObj);
844	AssertPtr(ppVirt);
845	AssertPtr(pHCPhys);
846	if (*pMemObj != NIL_RTR0MEMOBJ)
847	{
848	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
849	AssertRC(rc);
850	*pMemObj = NIL_RTR0MEMOBJ;
851	*ppVirt = 0;
852	*pHCPhys = 0;
853	}
854	}
855
856
857	/**
858	* Worker function to free VT-x related structures.
859	*
860	* @returns IPRT status code.
861	* @param pVM Pointer to the VM.
862	*/
863	static void hmR0VmxStructsFree(PVM pVM)
864	{
865	for (VMCPUID i = 0; i < pVM->cCpus; i++)
866	{
867	PVMCPU pVCpu = &pVM->aCpus[i];
868	AssertPtr(pVCpu);
869
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872
873	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
874	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
875
876	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
878	}
879
880	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
881	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
882	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
883	#endif
884	}
885
886
887	/**
888	* Worker function to allocate VT-x related VM structures.
889	*
890	* @returns IPRT status code.
891	* @param pVM Pointer to the VM.
892	*/
893	static int hmR0VmxStructsAlloc(PVM pVM)
894	{
895	/*
896	* Initialize members up-front so we can cleanup properly on allocation failure.
897	*/
898	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
899	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
900	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
901	pVM->hm.s.vmx.HCPhys##a_Name = 0;
902
903	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
904	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
905	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
906	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
907
908	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
909	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
910	#endif
911	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
912
913	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
914	for (VMCPUID i = 0; i < pVM->cCpus; i++)
915	{
916	PVMCPU pVCpu = &pVM->aCpus[i];
917	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
922	}
923	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
924	#undef VMXLOCAL_INIT_VM_MEMOBJ
925
926	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
927	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
928	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
929	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
930
931	/*
932	* Allocate all the VT-x structures.
933	*/
934	int rc = VINF_SUCCESS;
935	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
936	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
937	if (RT_FAILURE(rc))
938	goto cleanup;
939	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
940	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
941	#endif
942
943	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
944	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
945	{
946	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
947	&pVM->hm.s.vmx.HCPhysApicAccess);
948	if (RT_FAILURE(rc))
949	goto cleanup;
950	}
951
952	/*
953	* Initialize per-VCPU VT-x structures.
954	*/
955	for (VMCPUID i = 0; i < pVM->cCpus; i++)
956	{
957	PVMCPU pVCpu = &pVM->aCpus[i];
958	AssertPtr(pVCpu);
959
960	/* Allocate the VM control structure (VMCS). */
961	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
962	if (RT_FAILURE(rc))
963	goto cleanup;
964
965	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
966	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
967	{
968	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
969	&pVCpu->hm.s.vmx.HCPhysVirtApic);
970	if (RT_FAILURE(rc))
971	goto cleanup;
972	}
973
974	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
975	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
976	{
977	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
978	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
979	if (RT_FAILURE(rc))
980	goto cleanup;
981	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
982	}
983
984	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
985	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
986	if (RT_FAILURE(rc))
987	goto cleanup;
988
989	/* Allocate the VM-exit MSR-load page for the host MSRs. */
990	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	return VINF_SUCCESS;
996
997	cleanup:
998	hmR0VmxStructsFree(pVM);
999	return rc;
1000	}
1001
1002
1003	/**
1004	* Does global VT-x initialization (called during module initialization).
1005	*
1006	* @returns VBox status code.
1007	*/
1008	VMMR0DECL(int) VMXR0GlobalInit(void)
1009	{
1010	#ifdef HMVMX_USE_FUNCTION_TABLE
1011	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1012	# ifdef VBOX_STRICT
1013	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1014	Assert(g_apfnVMExitHandlers[i]);
1015	# endif
1016	#endif
1017	return VINF_SUCCESS;
1018	}
1019
1020
1021	/**
1022	* Does global VT-x termination (called during module termination).
1023	*/
1024	VMMR0DECL(void) VMXR0GlobalTerm()
1025	{
1026	/* Nothing to do currently. */
1027	}
1028
1029
1030	/**
1031	* Sets up and activates VT-x on the current CPU.
1032	*
1033	* @returns VBox status code.
1034	* @param pCpu Pointer to the global CPU info struct.
1035	* @param pVM Pointer to the VM (can be NULL after a host resume
1036	* operation).
1037	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1038	* fEnabledByHost is true).
1039	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1040	* @a fEnabledByHost is true).
1041	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1042	* enable VT-x on the host.
1043	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1044	*/
1045	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1046	void *pvMsrs)
1047	{
1048	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1049	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051
1052	/* Enable VT-x if it's not already enabled by the host. */
1053	if (!fEnabledByHost)
1054	{
1055	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1056	if (RT_FAILURE(rc))
1057	return rc;
1058	}
1059
1060	/*
1061	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1062	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1063	*/
1064	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1065	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1066	{
1067	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1068	pCpu->fFlushAsidBeforeUse = false;
1069	}
1070	else
1071	pCpu->fFlushAsidBeforeUse = true;
1072
1073	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1074	++pCpu->cTlbFlushes;
1075
1076	return VINF_SUCCESS;
1077	}
1078
1079
1080	/**
1081	* Deactivates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pCpu Pointer to the global CPU info struct.
1085	* @param pvCpuPage Pointer to the VMXON region.
1086	* @param HCPhysCpuPage Physical address of the VMXON region.
1087	*
1088	* @remarks This function should never be called when SUPR0EnableVTx() or
1089	* similar was used to enable VT-x on the host.
1090	*/
1091	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1092	{
1093	NOREF(pCpu);
1094	NOREF(pvCpuPage);
1095	NOREF(HCPhysCpuPage);
1096
1097	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1098	return hmR0VmxLeaveRootMode();
1099	}
1100
1101
1102	/**
1103	* Sets the permission bits for the specified MSR in the MSR bitmap.
1104	*
1105	* @param pVCpu Pointer to the VMCPU.
1106	* @param uMSR The MSR value.
1107	* @param enmRead Whether reading this MSR causes a VM-exit.
1108	* @param enmWrite Whether writing this MSR causes a VM-exit.
1109	*/
1110	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1111	{
1112	int32_t iBit;
1113	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1114
1115	/*
1116	* Layout:
1117	* 0x000 - 0x3ff - Low MSR read bits
1118	* 0x400 - 0x7ff - High MSR read bits
1119	* 0x800 - 0xbff - Low MSR write bits
1120	* 0xc00 - 0xfff - High MSR write bits
1121	*/
1122	if (uMsr <= 0x00001FFF)
1123	iBit = uMsr;
1124	else if ( uMsr >= 0xC0000000
1125	&& uMsr <= 0xC0001FFF)
1126	{
1127	iBit = (uMsr - 0xC0000000);
1128	pbMsrBitmap += 0x400;
1129	}
1130	else
1131	{
1132	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1133	return;
1134	}
1135
1136	Assert(iBit <= 0x1fff);
1137	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1138	ASMBitSet(pbMsrBitmap, iBit);
1139	else
1140	ASMBitClear(pbMsrBitmap, iBit);
1141
1142	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1143	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1144	else
1145	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1146	}
1147
1148
1149	#ifdef VBOX_STRICT
1150	/**
1151	* Gets the permission bits for the specified MSR in the MSR bitmap.
1152	*
1153	* @returns VBox status code.
1154	* @retval VINF_SUCCESS if the specified MSR is found.
1155	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1156	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1157	*
1158	* @param pVCpu Pointer to the VMCPU.
1159	* @param uMsr The MSR.
1160	* @param penmRead Where to store the read permissions.
1161	* @param penmWrite Where to store the write permissions.
1162	*/
1163	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1164	{
1165	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1166	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1167	int32_t iBit;
1168	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170	/* See hmR0VmxSetMsrPermission() for the layout. */
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if ( uMsr >= 0xC0000000
1174	&& uMsr <= 0xC0001FFF)
1175	{
1176	iBit = (uMsr - 0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	{
1181	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182	return VERR_NOT_SUPPORTED;
1183	}
1184
1185	Assert(iBit <= 0x1fff);
1186	if (ASMBitTest(pbMsrBitmap, iBit))
1187	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1188	else
1189	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1190
1191	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1192	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1193	else
1194	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1195	return VINF_SUCCESS;
1196	}
1197	#endif /* VBOX_STRICT */
1198
1199
1200	/**
1201	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1202	* area.
1203	*
1204	* @returns VBox status code.
1205	* @param pVCpu Pointer to the VMCPU.
1206	* @param cMsrs The number of MSRs.
1207	*/
1208	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1209	{
1210	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1211	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1212	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1213	{
1214	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1215	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1216	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1217	}
1218
1219	/* Update number of guest MSRs to load/store across the world-switch. */
1220	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1221	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222
1223	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1224	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1225
1226	/* Update the VCPU's copy of the MSR count. */
1227	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1228
1229	return VINF_SUCCESS;
1230	}
1231
1232
1233	/**
1234	* Adds a new (or updates the value of an existing) guest/host MSR
1235	* pair to be swapped during the world-switch as part of the
1236	* auto-load/store MSR area in the VMCS.
1237	*
1238	* @returns VBox status code.
1239	* @param pVCpu Pointer to the VMCPU.
1240	* @param uMsr The MSR.
1241	* @param uGuestMsr Value of the guest MSR.
1242	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1243	* necessary.
1244	*/
1245	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1246	{
1247	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1248	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1249	uint32_t i;
1250	for (i = 0; i < cMsrs; i++)
1251	{
1252	if (pGuestMsr->u32Msr == uMsr)
1253	break;
1254	pGuestMsr++;
1255	}
1256
1257	bool fAdded = false;
1258	if (i == cMsrs)
1259	{
1260	++cMsrs;
1261	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1262	AssertRCReturn(rc, rc);
1263
1264	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1265	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1266	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1267
1268	fAdded = true;
1269	}
1270
1271	/* Update the MSR values in the auto-load/store MSR area. */
1272	pGuestMsr->u32Msr = uMsr;
1273	pGuestMsr->u64Value = uGuestMsrValue;
1274
1275	/* Create/update the MSR slot in the host MSR area. */
1276	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1277	pHostMsr += i;
1278	pHostMsr->u32Msr = uMsr;
1279
1280	/*
1281	* Update the host MSR only when requested by the caller AND when we're
1282	* adding it to the auto-load/store area. Otherwise, it would have been
1283	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1284	*/
1285	if ( fAdded
1286	&& fUpdateHostMsr)
1287	{
1288	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1289	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1290	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1291	}
1292
1293	return VINF_SUCCESS;
1294	}
1295
1296
1297	/**
1298	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1299	* auto-load/store MSR area in the VMCS.
1300	*
1301	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1302	* area.
1303	*
1304	* @returns VBox status code.
1305	* @param pVCpu Pointer to the VMCPU.
1306	* @param uMsr The MSR.
1307	*/
1308	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1309	{
1310	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1311	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1312	for (uint32_t i = 0; i < cMsrs; i++)
1313	{
1314	/* Find the MSR. */
1315	if (pGuestMsr->u32Msr == uMsr)
1316	{
1317	/* If it's the last MSR, simply reduce the count. */
1318	if (i == cMsrs - 1)
1319	{
1320	--cMsrs;
1321	break;
1322	}
1323
1324	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1325	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1326	pLastGuestMsr += cMsrs;
1327	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1328	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1329
1330	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1331	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1332	pLastHostMsr += cMsrs;
1333	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1334	pHostMsr->u64Value = pLastHostMsr->u64Value;
1335	--cMsrs;
1336	break;
1337	}
1338	pGuestMsr++;
1339	}
1340
1341	/* Update the VMCS if the count changed (meaning the MSR was found). */
1342	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1343	{
1344	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1345	AssertRCReturn(rc, rc);
1346
1347	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1348	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1349	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1350	}
1351
1352	return VINF_SUCCESS;
1353	}
1354
1355
1356	/**
1357	* Checks if the specified guest MSR is part of the auto-load/store area in
1358	* the VMCS.
1359	*
1360	* @returns true if found, false otherwise.
1361	* @param pVCpu Pointer to the VMCPU.
1362	* @param uMsr The MSR to find.
1363	*/
1364	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1365	{
1366	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1367	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1368
1369	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1370	{
1371	if (pGuestMsr->u32Msr == uMsr)
1372	return true;
1373	}
1374	return false;
1375	}
1376
1377
1378	/**
1379	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1380	*
1381	* @param pVCpu Pointer to the VMCPU.
1382	*
1383	* @remarks No-long-jump zone!!!
1384	*/
1385	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1386	{
1387	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1388	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1389	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1390	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1391
1392	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1393	{
1394	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1395	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1396	}
1397
1398	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1399	}
1400
1401
1402	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1403	/**
1404	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1405	* perform lazy restoration of the host MSRs while leaving VT-x.
1406	*
1407	* @param pVCpu Pointer to the VMCPU.
1408	*
1409	* @remarks No-long-jump zone!!!
1410	*/
1411	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1412	{
1413	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1414
1415	/*
1416	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1417	*/
1418	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1419	{
1420	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1421	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1422	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1423	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1424	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1425	}
1426	}
1427
1428
1429	/**
1430	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1431	* lazily while leaving VT-x.
1432	*
1433	* @returns true if it does, false otherwise.
1434	* @param pVCpu Pointer to the VMCPU.
1435	* @param uMsr The MSR to check.
1436	*/
1437	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1438	{
1439	NOREF(pVCpu);
1440	switch (uMsr)
1441	{
1442	case MSR_K8_LSTAR:
1443	case MSR_K6_STAR:
1444	case MSR_K8_SF_MASK:
1445	case MSR_K8_KERNEL_GS_BASE:
1446	return true;
1447	}
1448	return false;
1449	}
1450
1451
1452	/**
1453	* Saves a set of guests MSRs back into the guest-CPU context.
1454	*
1455	* @param pVCpu Pointer to the VMCPU.
1456	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1457	* out-of-sync. Make sure to update the required fields
1458	* before using them.
1459	*
1460	* @remarks No-long-jump zone!!!
1461	*/
1462	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1463	{
1464	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1465	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1466
1467	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1468	{
1469	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1470	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1471	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1472	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1473	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1474	}
1475	}
1476
1477
1478	/**
1479	* Loads a set of guests MSRs to allow read/passthru to the guest.
1480	*
1481	* The name of this function is slightly confusing. This function does NOT
1482	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1483	* common prefix for functions dealing with "lazy restoration" of the shared
1484	* MSRs.
1485	*
1486	* @param pVCpu Pointer to the VMCPU.
1487	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1488	* out-of-sync. Make sure to update the required fields
1489	* before using them.
1490	*
1491	* @remarks No-long-jump zone!!!
1492	*/
1493	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1494	{
1495	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1497
1498	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1499	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1500	{
1501	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1502	do { \
1503	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1504	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1505	else \
1506	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1507	} while (0)
1508
1509	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1510	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1511	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1512	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1513	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1514	}
1515	else
1516	{
1517	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1518	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1519	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1520	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1521	}
1522	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1523	}
1524
1525
1526	/**
1527	* Performs lazy restoration of the set of host MSRs if they were previously
1528	* loaded with guest MSR values.
1529	*
1530	* @param pVCpu Pointer to the VMCPU.
1531	*
1532	* @remarks No-long-jump zone!!!
1533	* @remarks The guest MSRs should have been saved back into the guest-CPU
1534	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1535	*/
1536	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1537	{
1538	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1539	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1540
1541	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1542	{
1543	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1544	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1545	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1546	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1547	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1548	}
1549	pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST \| VMX_RESTORE_HOST_MSR_SAVED_HOST);
1550	}
1551	#endif /* HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
1552
1553
1554	#ifdef VBOX_STRICT
1555	/**
1556	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1557	* VMCS are correct.
1558	*
1559	* @param pVCpu Pointer to the VMCPU.
1560	*/
1561	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1562	{
1563	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1564
1565	/* Verify MSR counts in the VMCS are what we think it should be. */
1566	uint32_t cMsrs;
1567	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1568	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1569
1570	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1571	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1572
1573	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1574	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1575
1576	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1577	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1578	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1579	{
1580	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1581	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1582	pGuestMsr->u32Msr));
1583
1584	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1585	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1586	pHostMsr->u64Value, u64Msr));
1587
1588	/* Verify that the permissions are as expected in the MSR bitmap. */
1589	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1590	{
1591	VMXMSREXITREAD enmRead;
1592	VMXMSREXITWRITE enmWrite;
1593	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1594	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1595	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1596	pGuestMsr->u32Msr));
1597	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1598	pGuestMsr->u32Msr));
1599	}
1600	}
1601	}
1602	# endif /* VBOX_STRICT */
1603
1604
1605	/**
1606	* Flushes the TLB using EPT.
1607	*
1608	* @returns VBox status code.
1609	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1610	* enmFlush).
1611	* @param enmFlush Type of flush.
1612	*
1613	* @remarks Caller is responsible for making sure this function is called only
1614	* when NestedPaging is supported and providing @a enmFlush that is
1615	* supported by the CPU.
1616	* @remarks Can be called with interrupts disabled.
1617	*/
1618	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1619	{
1620	uint64_t au64Descriptor[2];
1621	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1622	au64Descriptor[0] = 0;
1623	else
1624	{
1625	Assert(pVCpu);
1626	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1627	}
1628	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1629
1630	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1631	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1632	rc));
1633	if ( RT_SUCCESS(rc)
1634	&& pVCpu)
1635	{
1636	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1637	}
1638	}
1639
1640
1641	/**
1642	* Flushes the TLB using VPID.
1643	*
1644	* @returns VBox status code.
1645	* @param pVM Pointer to the VM.
1646	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1647	* enmFlush).
1648	* @param enmFlush Type of flush.
1649	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1650	* on @a enmFlush).
1651	*
1652	* @remarks Can be called with interrupts disabled.
1653	*/
1654	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1655	{
1656	NOREF(pVM);
1657	AssertPtr(pVM);
1658	Assert(pVM->hm.s.vmx.fVpid);
1659
1660	uint64_t au64Descriptor[2];
1661	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1662	{
1663	au64Descriptor[0] = 0;
1664	au64Descriptor[1] = 0;
1665	}
1666	else
1667	{
1668	AssertPtr(pVCpu);
1669	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1670	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1671	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1672	au64Descriptor[1] = GCPtr;
1673	}
1674
1675	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1676	AssertMsg(rc == VINF_SUCCESS,
1677	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1678	if ( RT_SUCCESS(rc)
1679	&& pVCpu)
1680	{
1681	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1682	}
1683	}
1684
1685
1686	/**
1687	* Invalidates a guest page by guest virtual address. Only relevant for
1688	* EPT/VPID, otherwise there is nothing really to invalidate.
1689	*
1690	* @returns VBox status code.
1691	* @param pVM Pointer to the VM.
1692	* @param pVCpu Pointer to the VMCPU.
1693	* @param GCVirt Guest virtual address of the page to invalidate.
1694	*/
1695	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1696	{
1697	AssertPtr(pVM);
1698	AssertPtr(pVCpu);
1699	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1700
1701	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1702	if (!fFlushPending)
1703	{
1704	/*
1705	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1706	* See @bugref{6043} and @bugref{6177}.
1707	*
1708	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1709	* function maybe called in a loop with individual addresses.
1710	*/
1711	if (pVM->hm.s.vmx.fVpid)
1712	{
1713	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1714	{
1715	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1716	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1717	}
1718	else
1719	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1720	}
1721	else if (pVM->hm.s.fNestedPaging)
1722	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1723	}
1724
1725	return VINF_SUCCESS;
1726	}
1727
1728
1729	/**
1730	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1731	* otherwise there is nothing really to invalidate.
1732	*
1733	* @returns VBox status code.
1734	* @param pVM Pointer to the VM.
1735	* @param pVCpu Pointer to the VMCPU.
1736	* @param GCPhys Guest physical address of the page to invalidate.
1737	*/
1738	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1739	{
1740	NOREF(pVM); NOREF(GCPhys);
1741	LogFlowFunc(("%RGp\n", GCPhys));
1742
1743	/*
1744	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1745	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1746	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1747	*/
1748	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1749	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1750	return VINF_SUCCESS;
1751	}
1752
1753
1754	/**
1755	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1756	* case where neither EPT nor VPID is supported by the CPU.
1757	*
1758	* @param pVM Pointer to the VM.
1759	* @param pVCpu Pointer to the VMCPU.
1760	* @param pCpu Pointer to the global HM struct.
1761	*
1762	* @remarks Called with interrupts disabled.
1763	*/
1764	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1765	{
1766	AssertPtr(pVCpu);
1767	AssertPtr(pCpu);
1768	NOREF(pVM);
1769
1770	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1771
1772	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1773	#if 0
1774	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1775	pVCpu->hm.s.TlbShootdown.cPages = 0;
1776	#endif
1777
1778	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1779	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1780	pVCpu->hm.s.fForceTLBFlush = false;
1781	return;
1782	}
1783
1784
1785	/**
1786	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1787	*
1788	* @param pVM Pointer to the VM.
1789	* @param pVCpu Pointer to the VMCPU.
1790	* @param pCpu Pointer to the global HM CPU struct.
1791	* @remarks All references to "ASID" in this function pertains to "VPID" in
1792	* Intel's nomenclature. The reason is, to avoid confusion in compare
1793	* statements since the host-CPU copies are named "ASID".
1794	*
1795	* @remarks Called with interrupts disabled.
1796	*/
1797	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1798	{
1799	#ifdef VBOX_WITH_STATISTICS
1800	bool fTlbFlushed = false;
1801	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1802	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1803	if (!fTlbFlushed) \
1804	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1805	} while (0)
1806	#else
1807	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1808	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1809	#endif
1810
1811	AssertPtr(pVM);
1812	AssertPtr(pCpu);
1813	AssertPtr(pVCpu);
1814	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1815	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1816	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1817
1818
1819	/*
1820	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1821	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1822	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1823	*/
1824	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1825	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1826	{
1827	++pCpu->uCurrentAsid;
1828	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1829	{
1830	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1831	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1832	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1833	}
1834
1835	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1836	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1837	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1838
1839	/*
1840	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1841	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1842	*/
1843	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1844	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1845	HMVMX_SET_TAGGED_TLB_FLUSHED();
1846	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1847	}
1848
1849	/* Check for explicit TLB shootdowns. */
1850	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1851	{
1852	/*
1853	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1854	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1855	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1856	* but not guest-physical mappings.
1857	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1858	*/
1859	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1860	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1861	HMVMX_SET_TAGGED_TLB_FLUSHED();
1862	}
1863
1864	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1865	* where it is commented out. Support individual entry flushing
1866	* someday. */
1867	#if 0
1868	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1869	{
1870	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1871
1872	/*
1873	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1874	* as supported by the CPU.
1875	*/
1876	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1877	{
1878	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1879	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1880	}
1881	else
1882	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1883
1884	HMVMX_SET_TAGGED_TLB_FLUSHED();
1885	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1886	pVCpu->hm.s.TlbShootdown.cPages = 0;
1887	}
1888	#endif
1889
1890	pVCpu->hm.s.fForceTLBFlush = false;
1891
1892	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1893
1894	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1895	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1896	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1897	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1898	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1899	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1900	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1901	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1902
1903	/* Update VMCS with the VPID. */
1904	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1905	AssertRC(rc);
1906
1907	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1908	}
1909
1910
1911	/**
1912	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1913	*
1914	* @returns VBox status code.
1915	* @param pVM Pointer to the VM.
1916	* @param pVCpu Pointer to the VMCPU.
1917	* @param pCpu Pointer to the global HM CPU struct.
1918	*
1919	* @remarks Called with interrupts disabled.
1920	*/
1921	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1922	{
1923	AssertPtr(pVM);
1924	AssertPtr(pVCpu);
1925	AssertPtr(pCpu);
1926	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1927	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1928
1929	/*
1930	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1931	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1932	*/
1933	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1934	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1935	{
1936	pVCpu->hm.s.fForceTLBFlush = true;
1937	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1938	}
1939
1940	/* Check for explicit TLB shootdown flushes. */
1941	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1942	{
1943	pVCpu->hm.s.fForceTLBFlush = true;
1944	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1945	}
1946
1947	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1948	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1949
1950	if (pVCpu->hm.s.fForceTLBFlush)
1951	{
1952	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1953	pVCpu->hm.s.fForceTLBFlush = false;
1954	}
1955	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1956	* where it is commented out. Support individual entry flushing
1957	* someday. */
1958	#if 0
1959	else
1960	{
1961	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1962	{
1963	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1964	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1965	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1966	}
1967	else
1968	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1969
1970	pVCpu->hm.s.TlbShootdown.cPages = 0;
1971	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1972	}
1973	#endif
1974	}
1975
1976
1977	/**
1978	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1979	*
1980	* @returns VBox status code.
1981	* @param pVM Pointer to the VM.
1982	* @param pVCpu Pointer to the VMCPU.
1983	* @param pCpu Pointer to the global HM CPU struct.
1984	*
1985	* @remarks Called with interrupts disabled.
1986	*/
1987	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1988	{
1989	AssertPtr(pVM);
1990	AssertPtr(pVCpu);
1991	AssertPtr(pCpu);
1992	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1993	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1994
1995	/*
1996	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1997	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1998	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1999	*/
2000	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2001	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2002	{
2003	pVCpu->hm.s.fForceTLBFlush = true;
2004	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2005	}
2006
2007	/* Check for explicit TLB shootdown flushes. */
2008	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2009	{
2010	/*
2011	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2012	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2013	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2014	*/
2015	pVCpu->hm.s.fForceTLBFlush = true;
2016	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2017	}
2018
2019	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2020	if (pVCpu->hm.s.fForceTLBFlush)
2021	{
2022	++pCpu->uCurrentAsid;
2023	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2024	{
2025	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2026	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2027	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2028	}
2029
2030	pVCpu->hm.s.fForceTLBFlush = false;
2031	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2032	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2033	if (pCpu->fFlushAsidBeforeUse)
2034	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2035	}
2036	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2037	* where it is commented out. Support individual entry flushing
2038	* someday. */
2039	#if 0
2040	else
2041	{
2042	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2043	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2044	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2045	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2046
2047	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2048	{
2049	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2050	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2051	{
2052	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2053	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2054	}
2055	else
2056	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2057
2058	pVCpu->hm.s.TlbShootdown.cPages = 0;
2059	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2060	}
2061	else
2062	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2063	}
2064	#endif
2065
2066	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2067	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2068	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2069	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
2070	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2071	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2072
2073	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2074	AssertRC(rc);
2075	}
2076
2077
2078	/**
2079	* Flushes the guest TLB entry based on CPU capabilities.
2080	*
2081	* @param pVCpu Pointer to the VMCPU.
2082	* @param pCpu Pointer to the global HM CPU struct.
2083	*/
2084	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2085	{
2086	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2087	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2088	#endif
2089	PVM pVM = pVCpu->CTX_SUFF(pVM);
2090	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2091	{
2092	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2093	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2094	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2095	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2096	default:
2097	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2098	break;
2099	}
2100
2101	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2102	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2103
2104	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
2105	}
2106
2107
2108	/**
2109	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2110	* TLB entries from the host TLB before VM-entry.
2111	*
2112	* @returns VBox status code.
2113	* @param pVM Pointer to the VM.
2114	*/
2115	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2116	{
2117	/*
2118	* Determine optimal flush type for Nested Paging.
2119	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2120	* guest execution (see hmR3InitFinalizeR0()).
2121	*/
2122	if (pVM->hm.s.fNestedPaging)
2123	{
2124	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2125	{
2126	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2127	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2128	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2129	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2130	else
2131	{
2132	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2133	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2134	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2135	}
2136
2137	/* Make sure the write-back cacheable memory type for EPT is supported. */
2138	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2139	{
2140	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2141	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2142	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2143	}
2144	}
2145	else
2146	{
2147	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2148	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2149	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2150	}
2151	}
2152
2153	/*
2154	* Determine optimal flush type for VPID.
2155	*/
2156	if (pVM->hm.s.vmx.fVpid)
2157	{
2158	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2159	{
2160	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2161	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2162	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2163	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2164	else
2165	{
2166	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2167	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2168	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2169	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2170	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2171	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2172	pVM->hm.s.vmx.fVpid = false;
2173	}
2174	}
2175	else
2176	{
2177	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2178	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2179	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2180	pVM->hm.s.vmx.fVpid = false;
2181	}
2182	}
2183
2184	/*
2185	* Setup the handler for flushing tagged-TLBs.
2186	*/
2187	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2188	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2189	else if (pVM->hm.s.fNestedPaging)
2190	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2191	else if (pVM->hm.s.vmx.fVpid)
2192	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2193	else
2194	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2195	return VINF_SUCCESS;
2196	}
2197
2198
2199	/**
2200	* Sets up pin-based VM-execution controls in the VMCS.
2201	*
2202	* @returns VBox status code.
2203	* @param pVM Pointer to the VM.
2204	* @param pVCpu Pointer to the VMCPU.
2205	*/
2206	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2207	{
2208	AssertPtr(pVM);
2209	AssertPtr(pVCpu);
2210
2211	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2212	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2213
2214	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2215	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2216	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2217
2218	/* Enable the VMX preemption timer. */
2219	if (pVM->hm.s.vmx.fUsePreemptTimer)
2220	{
2221	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2222	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2223	}
2224
2225	if ((val & zap) != val)
2226	{
2227	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2228	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2229	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2230	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2231	}
2232
2233	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2234	AssertRCReturn(rc, rc);
2235
2236	/* Update VCPU with the currently set pin-based VM-execution controls. */
2237	pVCpu->hm.s.vmx.u32PinCtls = val;
2238	return rc;
2239	}
2240
2241
2242	/**
2243	* Sets up processor-based VM-execution controls in the VMCS.
2244	*
2245	* @returns VBox status code.
2246	* @param pVM Pointer to the VM.
2247	* @param pVMCPU Pointer to the VMCPU.
2248	*/
2249	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2250	{
2251	AssertPtr(pVM);
2252	AssertPtr(pVCpu);
2253
2254	int rc = VERR_INTERNAL_ERROR_5;
2255	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2256	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2257
2258	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2259	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2260	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2261	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2262	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2263	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2264	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2265
2266	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2267	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2268	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2269	{
2270	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2271	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2272	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2273	}
2274
2275	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2276	if (!pVM->hm.s.fNestedPaging)
2277	{
2278	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2279	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2280	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2281	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2282	}
2283
2284	/* Use TPR shadowing if supported by the CPU. */
2285	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2286	{
2287	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2288	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2289	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2290	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2291	AssertRCReturn(rc, rc);
2292
2293	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2294	/* CR8 writes causes a VM-exit based on TPR threshold. */
2295	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2296	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2297	}
2298	else
2299	{
2300	/*
2301	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2302	* Set this control only for 64-bit guests.
2303	*/
2304	if (pVM->hm.s.fAllow64BitGuests)
2305	{
2306	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2307	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2308	}
2309	}
2310
2311	/* Use MSR-bitmaps if supported by the CPU. */
2312	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2313	{
2314	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2315
2316	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2317	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2318	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2319	AssertRCReturn(rc, rc);
2320
2321	/*
2322	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2323	* automatically as dedicated fields in the VMCS.
2324	*/
2325	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2326	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2327	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2328	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2329	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2330
2331	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2332	/*
2333	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2334	*/
2335	if ( HMVMX_IS_64BIT_HOST_MODE()
2336	&& pVM->hm.s.fAllow64BitGuests)
2337	{
2338	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2339	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2340	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2341	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2342	}
2343	#endif
2344	}
2345
2346	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2347	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2348	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2349
2350	if ((val & zap) != val)
2351	{
2352	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2353	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2354	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2355	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2356	}
2357
2358	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2359	AssertRCReturn(rc, rc);
2360
2361	/* Update VCPU with the currently set processor-based VM-execution controls. */
2362	pVCpu->hm.s.vmx.u32ProcCtls = val;
2363
2364	/*
2365	* Secondary processor-based VM-execution controls.
2366	*/
2367	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2368	{
2369	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2370	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2371
2372	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2373	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2374
2375	if (pVM->hm.s.fNestedPaging)
2376	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2377	else
2378	{
2379	/*
2380	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2381	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2382	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2383	*/
2384	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2385	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2386	}
2387
2388	if (pVM->hm.s.vmx.fVpid)
2389	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2390
2391	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2392	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2393
2394	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2395	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2396	* done dynamically. */
2397	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2398	{
2399	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2400	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2401	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2402	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2403	AssertRCReturn(rc, rc);
2404	}
2405
2406	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2407	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2408
2409	if ((val & zap) != val)
2410	{
2411	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2412	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2413	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2414	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2415	}
2416
2417	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2418	AssertRCReturn(rc, rc);
2419
2420	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2421	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2422	}
2423	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2424	{
2425	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2426	"available\n"));
2427	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2428	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2429	}
2430
2431	return VINF_SUCCESS;
2432	}
2433
2434
2435	/**
2436	* Sets up miscellaneous (everything other than Pin & Processor-based
2437	* VM-execution) control fields in the VMCS.
2438	*
2439	* @returns VBox status code.
2440	* @param pVM Pointer to the VM.
2441	* @param pVCpu Pointer to the VMCPU.
2442	*/
2443	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2444	{
2445	NOREF(pVM);
2446	AssertPtr(pVM);
2447	AssertPtr(pVCpu);
2448
2449	int rc = VERR_GENERAL_FAILURE;
2450
2451	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2452	#if 0
2453	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2454	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2455	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2456
2457	/*
2458	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2459	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2460	* We thus use the exception bitmap to control it rather than use both.
2461	*/
2462	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2463	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2464
2465	/** @todo Explore possibility of using IO-bitmaps. */
2466	/* All IO & IOIO instructions cause VM-exits. */
2467	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2468	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2469
2470	/* Initialize the MSR-bitmap area. */
2471	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2472	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2473	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2474	#endif
2475
2476	/* Setup MSR auto-load/store area. */
2477	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2478	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2479	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2480	AssertRCReturn(rc, rc);
2481	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2482	AssertRCReturn(rc, rc);
2483
2484	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2485	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2486	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2487	AssertRCReturn(rc, rc);
2488
2489	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2490	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2491	AssertRCReturn(rc, rc);
2492
2493	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2494	#if 0
2495	/* Setup debug controls */
2496	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2497	AssertRCReturn(rc, rc);
2498	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2499	AssertRCReturn(rc, rc);
2500	#endif
2501
2502	return rc;
2503	}
2504
2505
2506	/**
2507	* Sets up the initial exception bitmap in the VMCS based on static conditions
2508	* (i.e. conditions that cannot ever change after starting the VM).
2509	*
2510	* @returns VBox status code.
2511	* @param pVM Pointer to the VM.
2512	* @param pVCpu Pointer to the VMCPU.
2513	*/
2514	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2515	{
2516	AssertPtr(pVM);
2517	AssertPtr(pVCpu);
2518
2519	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2520
2521	uint32_t u32XcptBitmap = 0;
2522
2523	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2524	if (!pVM->hm.s.fNestedPaging)
2525	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2526
2527	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2528	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2529	AssertRCReturn(rc, rc);
2530	return rc;
2531	}
2532
2533
2534	/**
2535	* Sets up the initial guest-state mask. The guest-state mask is consulted
2536	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2537	* for the nested virtualization case (as it would cause a VM-exit).
2538	*
2539	* @param pVCpu Pointer to the VMCPU.
2540	*/
2541	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2542	{
2543	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2544	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2545	return VINF_SUCCESS;
2546	}
2547
2548
2549	/**
2550	* Does per-VM VT-x initialization.
2551	*
2552	* @returns VBox status code.
2553	* @param pVM Pointer to the VM.
2554	*/
2555	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2556	{
2557	LogFlowFunc(("pVM=%p\n", pVM));
2558
2559	int rc = hmR0VmxStructsAlloc(pVM);
2560	if (RT_FAILURE(rc))
2561	{
2562	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2563	return rc;
2564	}
2565
2566	return VINF_SUCCESS;
2567	}
2568
2569
2570	/**
2571	* Does per-VM VT-x termination.
2572	*
2573	* @returns VBox status code.
2574	* @param pVM Pointer to the VM.
2575	*/
2576	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2577	{
2578	LogFlowFunc(("pVM=%p\n", pVM));
2579
2580	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2581	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2582	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2583	#endif
2584	hmR0VmxStructsFree(pVM);
2585	return VINF_SUCCESS;
2586	}
2587
2588
2589	/**
2590	* Sets up the VM for execution under VT-x.
2591	* This function is only called once per-VM during initialization.
2592	*
2593	* @returns VBox status code.
2594	* @param pVM Pointer to the VM.
2595	*/
2596	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2597	{
2598	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2599	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2600
2601	LogFlowFunc(("pVM=%p\n", pVM));
2602
2603	/*
2604	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2605	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2606	*/
2607	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2608	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2609	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2610	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2611	{
2612	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2613	return VERR_INTERNAL_ERROR;
2614	}
2615
2616	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2617	/*
2618	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2619	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2620	*/
2621	if ( pVM->hm.s.fAllow64BitGuests
2622	&& !HMVMX_IS_64BIT_HOST_MODE())
2623	{
2624	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2625	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2626	}
2627	#endif
2628
2629	/* Initialize these always, see hmR3InitFinalizeR0().*/
2630	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2631	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2632
2633	/* Setup the tagged-TLB flush handlers. */
2634	int rc = hmR0VmxSetupTaggedTlb(pVM);
2635	if (RT_FAILURE(rc))
2636	{
2637	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2638	return rc;
2639	}
2640
2641	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2642	{
2643	PVMCPU pVCpu = &pVM->aCpus[i];
2644	AssertPtr(pVCpu);
2645	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2646
2647	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2648	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2649
2650	/* Set revision dword at the beginning of the VMCS structure. */
2651	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2652
2653	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2654	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2655	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2656	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2657
2658	/* Load this VMCS as the current VMCS. */
2659	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2660	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2661	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2662
2663	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2664	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2665	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2666
2667	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2668	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2669	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2670
2671	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2672	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2673	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2674
2675	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2676	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2677	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2678
2679	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2680	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2681	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2682
2683	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2684	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2685	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2686	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2687	#endif
2688
2689	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2690	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2691	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2692	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2693
2694	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2695
2696	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2697	}
2698
2699	return VINF_SUCCESS;
2700	}
2701
2702
2703	/**
2704	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2705	* the VMCS.
2706	*
2707	* @returns VBox status code.
2708	* @param pVM Pointer to the VM.
2709	* @param pVCpu Pointer to the VMCPU.
2710	*/
2711	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2712	{
2713	NOREF(pVM); NOREF(pVCpu);
2714
2715	RTCCUINTREG uReg = ASMGetCR0();
2716	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2717	AssertRCReturn(rc, rc);
2718
2719	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2720	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2721	if (HMVMX_IS_64BIT_HOST_MODE())
2722	{
2723	uint64_t uRegCR3 = HMR0Get64bitCR3();
2724	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2725	}
2726	else
2727	#endif
2728	{
2729	uReg = ASMGetCR3();
2730	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2731	}
2732	AssertRCReturn(rc, rc);
2733
2734	uReg = ASMGetCR4();
2735	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2736	AssertRCReturn(rc, rc);
2737	return rc;
2738	}
2739
2740
2741	#if HC_ARCH_BITS == 64
2742	/**
2743	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2744	* requirements. See hmR0VmxSaveHostSegmentRegs().
2745	*/
2746	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2747	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2748	{ \
2749	bool fValidSelector = true; \
2750	if ((selValue) & X86_SEL_LDT) \
2751	{ \
2752	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2753	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2754	} \
2755	if (fValidSelector) \
2756	{ \
2757	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2758	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2759	} \
2760	(selValue) = 0; \
2761	}
2762	#endif
2763
2764
2765	/**
2766	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2767	* the host-state area in the VMCS.
2768	*
2769	* @returns VBox status code.
2770	* @param pVM Pointer to the VM.
2771	* @param pVCpu Pointer to the VMCPU.
2772	*/
2773	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2774	{
2775	NOREF(pVM);
2776	int rc = VERR_INTERNAL_ERROR_5;
2777
2778	/*
2779	* Host DS, ES, FS and GS segment registers.
2780	*/
2781	#if HC_ARCH_BITS == 64
2782	RTSEL uSelDS = ASMGetDS();
2783	RTSEL uSelES = ASMGetES();
2784	RTSEL uSelFS = ASMGetFS();
2785	RTSEL uSelGS = ASMGetGS();
2786	#else
2787	RTSEL uSelDS = 0;
2788	RTSEL uSelES = 0;
2789	RTSEL uSelFS = 0;
2790	RTSEL uSelGS = 0;
2791	#endif
2792
2793	/* Recalculate which host-state bits need to be manually restored. */
2794	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2795
2796	/*
2797	* Host CS and SS segment registers.
2798	*/
2799	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2800	RTSEL uSelCS;
2801	RTSEL uSelSS;
2802	if (HMVMX_IS_64BIT_HOST_MODE())
2803	{
2804	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2805	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2806	}
2807	else
2808	{
2809	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2810	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2811	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2812	}
2813	#else
2814	RTSEL uSelCS = ASMGetCS();
2815	RTSEL uSelSS = ASMGetSS();
2816	#endif
2817
2818	/*
2819	* Host TR segment register.
2820	*/
2821	RTSEL uSelTR = ASMGetTR();
2822
2823	#if HC_ARCH_BITS == 64
2824	/*
2825	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2826	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2827	*/
2828	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2829	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2830	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2831	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2832	# undef VMXLOCAL_ADJUST_HOST_SEG
2833	#endif
2834
2835	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2836	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2837	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2838	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2839	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2840	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2841	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2842	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2843	Assert(uSelCS);
2844	Assert(uSelTR);
2845
2846	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2847	#if 0
2848	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2849	Assert(uSelSS != 0);
2850	#endif
2851
2852	/* Write these host selector fields into the host-state area in the VMCS. */
2853	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2854	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2855	#if HC_ARCH_BITS == 64
2856	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2857	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2858	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2859	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2860	#endif
2861	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2862
2863	/*
2864	* Host GDTR and IDTR.
2865	*/
2866	RTGDTR Gdtr;
2867	RT_ZERO(Gdtr);
2868	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2869	if (HMVMX_IS_64BIT_HOST_MODE())
2870	{
2871	X86XDTR64 Gdtr64;
2872	X86XDTR64 Idtr64;
2873	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2874	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2875	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2876
2877	Gdtr.cbGdt = Gdtr64.cb;
2878	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2879	}
2880	else
2881	#endif
2882	{
2883	RTIDTR Idtr;
2884	ASMGetGDTR(&Gdtr);
2885	ASMGetIDTR(&Idtr);
2886	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2887	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2888
2889	#if HC_ARCH_BITS == 64
2890	/*
2891	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2892	* maximum limit (0xffff) on every VM-exit.
2893	*/
2894	if (Gdtr.cbGdt != 0xffff)
2895	{
2896	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2897	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2898	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2899	}
2900
2901	/*
2902	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2903	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2904	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2905	*/
2906	if (Idtr.cbIdt < 0x0fff)
2907	{
2908	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2909	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2910	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2911	}
2912	#endif
2913	}
2914
2915	/*
2916	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2917	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2918	*/
2919	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2920	{
2921	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2922	return VERR_VMX_INVALID_HOST_STATE;
2923	}
2924
2925	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2926	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2927	if (HMVMX_IS_64BIT_HOST_MODE())
2928	{
2929	/* We need the 64-bit TR base for hybrid darwin. */
2930	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2931	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2932	}
2933	else
2934	#endif
2935	{
2936	uintptr_t uTRBase;
2937	#if HC_ARCH_BITS == 64
2938	uTRBase = X86DESC64_BASE(pDesc);
2939
2940	/*
2941	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2942	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2943	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2944	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2945	*
2946	* [1] See Intel spec. 3.5 "System Descriptor Types".
2947	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2948	*/
2949	Assert(pDesc->System.u4Type == 11);
2950	if ( pDesc->System.u16LimitLow != 0x67
2951	\|\| pDesc->System.u4LimitHigh)
2952	{
2953	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2954	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2955
2956	/* Store the GDTR here as we need it while restoring TR. */
2957	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2958	}
2959	#else
2960	uTRBase = X86DESC_BASE(pDesc);
2961	#endif
2962	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2963	}
2964	AssertRCReturn(rc, rc);
2965
2966	/*
2967	* Host FS base and GS base.
2968	*/
2969	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2970	if (HMVMX_IS_64BIT_HOST_MODE())
2971	{
2972	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2973	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2974	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2975	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2976
2977	# if HC_ARCH_BITS == 64
2978	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2979	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2980	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2981	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2982	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2983	# endif
2984	}
2985	#endif
2986	return rc;
2987	}
2988
2989
2990	/**
2991	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2992	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2993	* the host after every successful VM-exit.
2994	*
2995	* @returns VBox status code.
2996	* @param pVM Pointer to the VM.
2997	* @param pVCpu Pointer to the VMCPU.
2998	*
2999	* @remarks No-long-jump zone!!!
3000	*/
3001	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3002	{
3003	NOREF(pVM);
3004
3005	AssertPtr(pVCpu);
3006	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3007
3008	int rc = VINF_SUCCESS;
3009	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3010	if ( HMVMX_IS_64BIT_HOST_MODE()
3011	&& pVM->hm.s.fAllow64BitGuests)
3012	{
3013	hmR0VmxLazySaveHostMsrs(pVCpu);
3014	}
3015	#endif
3016
3017	if (pVCpu->hm.s.vmx.cMsrs > 0)
3018	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
3019
3020	/*
3021	* Host Sysenter MSRs.
3022	*/
3023	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3024	AssertRCReturn(rc, rc);
3025	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3026	if (HMVMX_IS_64BIT_HOST_MODE())
3027	{
3028	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3029	AssertRCReturn(rc, rc);
3030	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3031	}
3032	else
3033	{
3034	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3035	AssertRCReturn(rc, rc);
3036	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3037	}
3038	#elif HC_ARCH_BITS == 32
3039	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3040	AssertRCReturn(rc, rc);
3041	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3042	#else
3043	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3044	AssertRCReturn(rc, rc);
3045	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3046	#endif
3047	AssertRCReturn(rc, rc);
3048
3049	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3050	* hmR0VmxSetupExitCtls() !! */
3051	return rc;
3052	}
3053
3054
3055	/**
3056	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3057	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3058	* controls".
3059	*
3060	* @returns VBox status code.
3061	* @param pVCpu Pointer to the VMCPU.
3062	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3063	* out-of-sync. Make sure to update the required fields
3064	* before using them.
3065	*
3066	* @remarks No-long-jump zone!!!
3067	*/
3068	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3069	{
3070	int rc = VINF_SUCCESS;
3071	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3072	{
3073	PVM pVM = pVCpu->CTX_SUFF(pVM);
3074	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3075	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3076
3077	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3078	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3079
3080	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3081	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3082	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3083	else
3084	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3085
3086	/*
3087	* The following should -not- be set (since we're not in SMM mode):
3088	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3089	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3090	*/
3091
3092	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3093	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3094	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3095
3096	if ((val & zap) != val)
3097	{
3098	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3099	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3100	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3101	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3102	}
3103
3104	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3105	AssertRCReturn(rc, rc);
3106
3107	/* Update VCPU with the currently set VM-exit controls. */
3108	pVCpu->hm.s.vmx.u32EntryCtls = val;
3109	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3110	}
3111	return rc;
3112	}
3113
3114
3115	/**
3116	* Sets up the VM-exit controls in the VMCS.
3117	*
3118	* @returns VBox status code.
3119	* @param pVM Pointer to the VM.
3120	* @param pVCpu Pointer to the VMCPU.
3121	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3122	* out-of-sync. Make sure to update the required fields
3123	* before using them.
3124	*
3125	* @remarks requires EFER.
3126	*/
3127	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3128	{
3129	NOREF(pMixedCtx);
3130
3131	int rc = VINF_SUCCESS;
3132	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3133	{
3134	PVM pVM = pVCpu->CTX_SUFF(pVM);
3135	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3136	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3137
3138	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3139	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3140
3141	/*
3142	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3143	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3144	*/
3145	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3146	if (HMVMX_IS_64BIT_HOST_MODE())
3147	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3148	else
3149	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3150	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3151	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3152	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3153	else
3154	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3155	#endif
3156
3157	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3158	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3159
3160	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3161	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3162	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3163	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3164	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3165
3166	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3167	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3168
3169	if ((val & zap) != val)
3170	{
3171	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3172	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3173	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3174	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3175	}
3176
3177	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3178	AssertRCReturn(rc, rc);
3179
3180	/* Update VCPU with the currently set VM-exit controls. */
3181	pVCpu->hm.s.vmx.u32ExitCtls = val;
3182	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3183	}
3184	return rc;
3185	}
3186
3187
3188	/**
3189	* Loads the guest APIC and related state.
3190	*
3191	* @returns VBox status code.
3192	* @param pVM Pointer to the VM.
3193	* @param pVCpu Pointer to the VMCPU.
3194	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3195	* out-of-sync. Make sure to update the required fields
3196	* before using them.
3197	*/
3198	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3199	{
3200	NOREF(pMixedCtx);
3201
3202	int rc = VINF_SUCCESS;
3203	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3204	{
3205	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3206	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3207	{
3208	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3209
3210	bool fPendingIntr = false;
3211	uint8_t u8Tpr = 0;
3212	uint8_t u8PendingIntr = 0;
3213	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3214	AssertRCReturn(rc, rc);
3215
3216	/*
3217	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3218	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3219	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3220	* the interrupt when we VM-exit for other reasons.
3221	*/
3222	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3223	uint32_t u32TprThreshold = 0;
3224	if (fPendingIntr)
3225	{
3226	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3227	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3228	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3229	if (u8PendingPriority <= u8TprPriority)
3230	u32TprThreshold = u8PendingPriority;
3231	else
3232	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3233	}
3234	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3235
3236	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3237	AssertRCReturn(rc, rc);
3238	}
3239
3240	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3241	}
3242	return rc;
3243	}
3244
3245
3246	/**
3247	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3248	*
3249	* @returns Guest's interruptibility-state.
3250	* @param pVCpu Pointer to the VMCPU.
3251	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3252	* out-of-sync. Make sure to update the required fields
3253	* before using them.
3254	*
3255	* @remarks No-long-jump zone!!!
3256	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3257	*/
3258	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3259	{
3260	/*
3261	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3262	* inhibit interrupts or clear any existing interrupt-inhibition.
3263	*/
3264	uint32_t uIntrState = 0;
3265	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3266	{
3267	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3268	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
3269	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
3270	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3271	{
3272	/*
3273	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3274	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3275	*/
3276	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3277	}
3278	else if (pMixedCtx->eflags.Bits.u1IF)
3279	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3280	else
3281	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3282	}
3283	return uIntrState;
3284	}
3285
3286
3287	/**
3288	* Loads the guest's interruptibility-state into the guest-state area in the
3289	* VMCS.
3290	*
3291	* @returns VBox status code.
3292	* @param pVCpu Pointer to the VMCPU.
3293	* @param uIntrState The interruptibility-state to set.
3294	*/
3295	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3296	{
3297	NOREF(pVCpu);
3298	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3299	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3300	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3301	AssertRCReturn(rc, rc);
3302	return rc;
3303	}
3304
3305
3306	/**
3307	* Loads the guest's RIP into the guest-state area in the VMCS.
3308	*
3309	* @returns VBox status code.
3310	* @param pVCpu Pointer to the VMCPU.
3311	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3312	* out-of-sync. Make sure to update the required fields
3313	* before using them.
3314	*
3315	* @remarks No-long-jump zone!!!
3316	*/
3317	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3318	{
3319	int rc = VINF_SUCCESS;
3320	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3321	{
3322	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3323	AssertRCReturn(rc, rc);
3324
3325	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3326	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3327	}
3328	return rc;
3329	}
3330
3331
3332	/**
3333	* Loads the guest's RSP into the guest-state area in the VMCS.
3334	*
3335	* @returns VBox status code.
3336	* @param pVCpu Pointer to the VMCPU.
3337	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3338	* out-of-sync. Make sure to update the required fields
3339	* before using them.
3340	*
3341	* @remarks No-long-jump zone!!!
3342	*/
3343	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3344	{
3345	int rc = VINF_SUCCESS;
3346	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3347	{
3348	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3349	AssertRCReturn(rc, rc);
3350
3351	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3352	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3353	}
3354	return rc;
3355	}
3356
3357
3358	/**
3359	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3360	*
3361	* @returns VBox status code.
3362	* @param pVCpu Pointer to the VMCPU.
3363	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3364	* out-of-sync. Make sure to update the required fields
3365	* before using them.
3366	*
3367	* @remarks No-long-jump zone!!!
3368	*/
3369	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3370	{
3371	int rc = VINF_SUCCESS;
3372	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3373	{
3374	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3375	Let us assert it as such and use 32-bit VMWRITE. */
3376	Assert(!(pMixedCtx->rflags.u64 >> 32));
3377	X86EFLAGS Eflags = pMixedCtx->eflags;
3378	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3379	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3380
3381	/*
3382	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3383	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3384	*/
3385	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3386	{
3387	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3388	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3389	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3390	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3391	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3392	}
3393
3394	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3395	AssertRCReturn(rc, rc);
3396
3397	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3398	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3399	}
3400	return rc;
3401	}
3402
3403
3404	/**
3405	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3406	*
3407	* @returns VBox status code.
3408	* @param pVCpu Pointer to the VMCPU.
3409	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3410	* out-of-sync. Make sure to update the required fields
3411	* before using them.
3412	*
3413	* @remarks No-long-jump zone!!!
3414	*/
3415	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3416	{
3417	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3418	AssertRCReturn(rc, rc);
3419	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3420	AssertRCReturn(rc, rc);
3421	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3422	AssertRCReturn(rc, rc);
3423	return rc;
3424	}
3425
3426
3427	/**
3428	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3429	* CR0 is partially shared with the host and we have to consider the FPU bits.
3430	*
3431	* @returns VBox status code.
3432	* @param pVM Pointer to the VM.
3433	* @param pVCpu Pointer to the VMCPU.
3434	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3435	* out-of-sync. Make sure to update the required fields
3436	* before using them.
3437	*
3438	* @remarks No-long-jump zone!!!
3439	*/
3440	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3441	{
3442	/*
3443	* Guest CR0.
3444	* Guest FPU.
3445	*/
3446	int rc = VINF_SUCCESS;
3447	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3448	{
3449	Assert(!(pMixedCtx->cr0 >> 32));
3450	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3451	PVM pVM = pVCpu->CTX_SUFF(pVM);
3452
3453	/* The guest's view (read access) of its CR0 is unblemished. */
3454	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3455	AssertRCReturn(rc, rc);
3456	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3457
3458	/* Setup VT-x's view of the guest CR0. */
3459	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3460	if (pVM->hm.s.fNestedPaging)
3461	{
3462	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3463	{
3464	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3465	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3466	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3467	}
3468	else
3469	{
3470	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3471	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3472	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3473	}
3474
3475	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3476	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3477	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3478
3479	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3480	AssertRCReturn(rc, rc);
3481	}
3482	else
3483	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3484
3485	/*
3486	* Guest FPU bits.
3487	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3488	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3489	*/
3490	u32GuestCR0 \|= X86_CR0_NE;
3491	bool fInterceptNM = false;
3492	if (CPUMIsGuestFPUStateActive(pVCpu))
3493	{
3494	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3495	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3496	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3497	}
3498	else
3499	{
3500	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3501	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3502	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3503	}
3504
3505	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3506	bool fInterceptMF = false;
3507	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3508	fInterceptMF = true;
3509
3510	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3511	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3512	{
3513	Assert(PDMVmmDevHeapIsEnabled(pVM));
3514	Assert(pVM->hm.s.vmx.pRealModeTSS);
3515	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3516	fInterceptNM = true;
3517	fInterceptMF = true;
3518	}
3519	else
3520	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3521
3522	if (fInterceptNM)
3523	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3524	else
3525	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3526
3527	if (fInterceptMF)
3528	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3529	else
3530	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3531
3532	/* Additional intercepts for debugging, define these yourself explicitly. */
3533	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3534	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3535	\| RT_BIT(X86_XCPT_BP)
3536	\| RT_BIT(X86_XCPT_DB)
3537	\| RT_BIT(X86_XCPT_DE)
3538	\| RT_BIT(X86_XCPT_NM)
3539	\| RT_BIT(X86_XCPT_UD)
3540	\| RT_BIT(X86_XCPT_NP)
3541	\| RT_BIT(X86_XCPT_SS)
3542	\| RT_BIT(X86_XCPT_GP)
3543	\| RT_BIT(X86_XCPT_PF)
3544	\| RT_BIT(X86_XCPT_MF)
3545	;
3546	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3547	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3548	#endif
3549
3550	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3551
3552	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3553	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3554	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3555	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3556	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3557	else
3558	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3559
3560	u32GuestCR0 \|= uSetCR0;
3561	u32GuestCR0 &= uZapCR0;
3562	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3563
3564	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3565	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3566	AssertRCReturn(rc, rc);
3567	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3568	AssertRCReturn(rc, rc);
3569	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3570
3571	/*
3572	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3573	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3574	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3575	*/
3576	uint32_t u32CR0Mask = 0;
3577	u32CR0Mask = X86_CR0_PE
3578	\| X86_CR0_NE
3579	\| X86_CR0_WP
3580	\| X86_CR0_PG
3581	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3582	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3583	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3584
3585	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3586	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3587	* and @bugref{6944}. */
3588	#if 0
3589	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3590	u32CR0Mask &= ~X86_CR0_PE;
3591	#endif
3592	if (pVM->hm.s.fNestedPaging)
3593	u32CR0Mask &= ~X86_CR0_WP;
3594
3595	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3596	if (fInterceptNM)
3597	{
3598	u32CR0Mask \|= X86_CR0_TS
3599	\| X86_CR0_MP;
3600	}
3601
3602	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3603	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3604	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3605	AssertRCReturn(rc, rc);
3606	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3607
3608	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3609	}
3610	return rc;
3611	}
3612
3613
3614	/**
3615	* Loads the guest control registers (CR3, CR4) into the guest-state area
3616	* in the VMCS.
3617	*
3618	* @returns VBox status code.
3619	* @param pVM Pointer to the VM.
3620	* @param pVCpu Pointer to the VMCPU.
3621	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3622	* out-of-sync. Make sure to update the required fields
3623	* before using them.
3624	*
3625	* @remarks No-long-jump zone!!!
3626	*/
3627	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3628	{
3629	int rc = VINF_SUCCESS;
3630	PVM pVM = pVCpu->CTX_SUFF(pVM);
3631
3632	/*
3633	* Guest CR2.
3634	* It's always loaded in the assembler code. Nothing to do here.
3635	*/
3636
3637	/*
3638	* Guest CR3.
3639	*/
3640	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3641	{
3642	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3643	if (pVM->hm.s.fNestedPaging)
3644	{
3645	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3646
3647	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3648	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3649	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3650	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3651
3652	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3653	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3654	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3655
3656	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3657	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3658	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3659	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3660
3661	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3662	AssertRCReturn(rc, rc);
3663	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3664
3665	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3666	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3667	{
3668	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3669	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3670	{
3671	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3672	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3673	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3674	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3675	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3676	}
3677
3678	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3679	have Unrestricted Execution to handle the guest when it's not using paging. */
3680	GCPhysGuestCR3 = pMixedCtx->cr3;
3681	}
3682	else
3683	{
3684	/*
3685	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3686	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3687	* EPT takes care of translating it to host-physical addresses.
3688	*/
3689	RTGCPHYS GCPhys;
3690	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3691	Assert(PDMVmmDevHeapIsEnabled(pVM));
3692
3693	/* We obtain it here every time as the guest could have relocated this PCI region. */
3694	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3695	AssertRCReturn(rc, rc);
3696
3697	GCPhysGuestCR3 = GCPhys;
3698	}
3699
3700	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3701	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3702	}
3703	else
3704	{
3705	/* Non-nested paging case, just use the hypervisor's CR3. */
3706	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3707
3708	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3709	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3710	}
3711	AssertRCReturn(rc, rc);
3712
3713	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3714	}
3715
3716	/*
3717	* Guest CR4.
3718	*/
3719	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3720	{
3721	Assert(!(pMixedCtx->cr4 >> 32));
3722	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3723
3724	/* The guest's view of its CR4 is unblemished. */
3725	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3726	AssertRCReturn(rc, rc);
3727	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3728
3729	/* Setup VT-x's view of the guest CR4. */
3730	/*
3731	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3732	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3733	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3734	*/
3735	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3736	{
3737	Assert(pVM->hm.s.vmx.pRealModeTSS);
3738	Assert(PDMVmmDevHeapIsEnabled(pVM));
3739	u32GuestCR4 &= ~X86_CR4_VME;
3740	}
3741
3742	if (pVM->hm.s.fNestedPaging)
3743	{
3744	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3745	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3746	{
3747	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3748	u32GuestCR4 \|= X86_CR4_PSE;
3749	/* Our identity mapping is a 32-bit page directory. */
3750	u32GuestCR4 &= ~X86_CR4_PAE;
3751	}
3752	/* else use guest CR4.*/
3753	}
3754	else
3755	{
3756	/*
3757	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3758	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3759	*/
3760	switch (pVCpu->hm.s.enmShadowMode)
3761	{
3762	case PGMMODE_REAL: /* Real-mode. */
3763	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3764	case PGMMODE_32_BIT: /* 32-bit paging. */
3765	{
3766	u32GuestCR4 &= ~X86_CR4_PAE;
3767	break;
3768	}
3769
3770	case PGMMODE_PAE: /* PAE paging. */
3771	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3772	{
3773	u32GuestCR4 \|= X86_CR4_PAE;
3774	break;
3775	}
3776
3777	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3778	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3779	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3780	break;
3781	#endif
3782	default:
3783	AssertFailed();
3784	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3785	}
3786	}
3787
3788	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3789	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3790	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3791	u32GuestCR4 \|= uSetCR4;
3792	u32GuestCR4 &= uZapCR4;
3793
3794	/* Write VT-x's view of the guest CR4 into the VMCS. */
3795	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3796	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3797	AssertRCReturn(rc, rc);
3798
3799	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3800	uint32_t u32CR4Mask = 0;
3801	u32CR4Mask = X86_CR4_VME
3802	\| X86_CR4_PAE
3803	\| X86_CR4_PGE
3804	\| X86_CR4_PSE
3805	\| X86_CR4_VMXE;
3806	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3807	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3808	AssertRCReturn(rc, rc);
3809
3810	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3811	}
3812	return rc;
3813	}
3814
3815
3816	/**
3817	* Loads the guest debug registers into the guest-state area in the VMCS.
3818	* This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3819	*
3820	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3821	*
3822	* @returns VBox status code.
3823	* @param pVCpu Pointer to the VMCPU.
3824	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3825	* out-of-sync. Make sure to update the required fields
3826	* before using them.
3827	*
3828	* @remarks No-long-jump zone!!!
3829	*/
3830	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3831	{
3832	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3833	return VINF_SUCCESS;
3834
3835	#ifdef VBOX_STRICT
3836	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3837	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3838	{
3839	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3840	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3841	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3842	}
3843	#endif
3844
3845	int rc;
3846	PVM pVM = pVCpu->CTX_SUFF(pVM);
3847	bool fInterceptDB = false;
3848	bool fInterceptMovDRx = false;
3849	if ( pVCpu->hm.s.fSingleInstruction
3850	\|\| DBGFIsStepping(pVCpu))
3851	{
3852	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3853	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3854	{
3855	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3856	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3857	AssertRCReturn(rc, rc);
3858	Assert(fInterceptDB == false);
3859	}
3860	else
3861	{
3862	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3863	pVCpu->hm.s.fClearTrapFlag = true;
3864	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3865	fInterceptDB = true;
3866	}
3867	}
3868
3869	if ( fInterceptDB
3870	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3871	{
3872	/*
3873	* Use the combined guest and host DRx values found in the hypervisor
3874	* register set because the debugger has breakpoints active or someone
3875	* is single stepping on the host side without a monitor trap flag.
3876	*
3877	* Note! DBGF expects a clean DR6 state before executing guest code.
3878	*/
3879	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3880	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3881	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3882	{
3883	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3884	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3885	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3886	}
3887	else
3888	#endif
3889	if (!CPUMIsHyperDebugStateActive(pVCpu))
3890	{
3891	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3892	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3893	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3894	}
3895
3896	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3897	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3898	AssertRCReturn(rc, rc);
3899
3900	pVCpu->hm.s.fUsingHyperDR7 = true;
3901	fInterceptDB = true;
3902	fInterceptMovDRx = true;
3903	}
3904	else
3905	{
3906	/*
3907	* If the guest has enabled debug registers, we need to load them prior to
3908	* executing guest code so they'll trigger at the right time.
3909	*/
3910	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3911	{
3912	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3913	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3914	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3915	{
3916	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3917	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3918	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3919	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3920	}
3921	else
3922	#endif
3923	if (!CPUMIsGuestDebugStateActive(pVCpu))
3924	{
3925	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3926	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3927	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3928	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3929	}
3930	Assert(!fInterceptDB);
3931	Assert(!fInterceptMovDRx);
3932	}
3933	/*
3934	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3935	* must intercept #DB in order to maintain a correct DR6 guest value.
3936	*/
3937	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3938	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3939	&& !CPUMIsGuestDebugStateActive(pVCpu))
3940	#else
3941	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3942	#endif
3943	{
3944	fInterceptMovDRx = true;
3945	fInterceptDB = true;
3946	}
3947
3948	/* Update guest DR7. */
3949	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3950	AssertRCReturn(rc, rc);
3951
3952	pVCpu->hm.s.fUsingHyperDR7 = false;
3953	}
3954
3955	/*
3956	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3957	*/
3958	if (fInterceptDB)
3959	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3960	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3961	{
3962	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3963	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3964	#endif
3965	}
3966	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3967	AssertRCReturn(rc, rc);
3968
3969	/*
3970	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3971	*/
3972	if (fInterceptMovDRx)
3973	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3974	else
3975	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3976	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3977	AssertRCReturn(rc, rc);
3978
3979	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3980	return VINF_SUCCESS;
3981	}
3982
3983
3984	#ifdef VBOX_STRICT
3985	/**
3986	* Strict function to validate segment registers.
3987	*
3988	* @remarks ASSUMES CR0 is up to date.
3989	*/
3990	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3991	{
3992	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3993	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3994	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3995	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3996	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3997	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3998	{
3999	/* Protected mode checks */
4000	/* CS */
4001	Assert(pCtx->cs.Attr.n.u1Present);
4002	Assert(!(pCtx->cs.Attr.u & 0xf00));
4003	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4004	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4005	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4006	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4007	\|\| (pCtx->cs.Attr.n.u1Granularity));
4008	/* CS cannot be loaded with NULL in protected mode. */
4009	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4010	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4011	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4012	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4013	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4014	else
4015	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4016	/* SS */
4017	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4018	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4019	if ( !(pCtx->cr0 & X86_CR0_PE)
4020	\|\| pCtx->cs.Attr.n.u4Type == 3)
4021	{
4022	Assert(!pCtx->ss.Attr.n.u2Dpl);
4023	}
4024	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4025	{
4026	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4027	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4028	Assert(pCtx->ss.Attr.n.u1Present);
4029	Assert(!(pCtx->ss.Attr.u & 0xf00));
4030	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4031	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4032	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4033	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4034	\|\| (pCtx->ss.Attr.n.u1Granularity));
4035	}
4036	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4037	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4038	{
4039	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4040	Assert(pCtx->ds.Attr.n.u1Present);
4041	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4042	Assert(!(pCtx->ds.Attr.u & 0xf00));
4043	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4044	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4045	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4046	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4047	\|\| (pCtx->ds.Attr.n.u1Granularity));
4048	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4049	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4050	}
4051	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4052	{
4053	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4054	Assert(pCtx->es.Attr.n.u1Present);
4055	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4056	Assert(!(pCtx->es.Attr.u & 0xf00));
4057	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4058	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4059	\|\| !(pCtx->es.Attr.n.u1Granularity));
4060	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4061	\|\| (pCtx->es.Attr.n.u1Granularity));
4062	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4063	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4064	}
4065	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4066	{
4067	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4068	Assert(pCtx->fs.Attr.n.u1Present);
4069	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4070	Assert(!(pCtx->fs.Attr.u & 0xf00));
4071	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4072	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4073	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4074	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4075	\|\| (pCtx->fs.Attr.n.u1Granularity));
4076	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4077	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4078	}
4079	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4080	{
4081	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4082	Assert(pCtx->gs.Attr.n.u1Present);
4083	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4084	Assert(!(pCtx->gs.Attr.u & 0xf00));
4085	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4086	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4087	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4088	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4089	\|\| (pCtx->gs.Attr.n.u1Granularity));
4090	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4091	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4092	}
4093	/* 64-bit capable CPUs. */
4094	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4095	Assert(!(pCtx->cs.u64Base >> 32));
4096	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4097	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4098	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4099	# endif
4100	}
4101	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4102	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4103	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4104	{
4105	/* Real and v86 mode checks. */
4106	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4107	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4108	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4109	{
4110	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4111	}
4112	else
4113	{
4114	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4115	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4116	}
4117
4118	/* CS */
4119	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4120	Assert(pCtx->cs.u32Limit == 0xffff);
4121	Assert(u32CSAttr == 0xf3);
4122	/* SS */
4123	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4124	Assert(pCtx->ss.u32Limit == 0xffff);
4125	Assert(u32SSAttr == 0xf3);
4126	/* DS */
4127	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4128	Assert(pCtx->ds.u32Limit == 0xffff);
4129	Assert(u32DSAttr == 0xf3);
4130	/* ES */
4131	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4132	Assert(pCtx->es.u32Limit == 0xffff);
4133	Assert(u32ESAttr == 0xf3);
4134	/* FS */
4135	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4136	Assert(pCtx->fs.u32Limit == 0xffff);
4137	Assert(u32FSAttr == 0xf3);
4138	/* GS */
4139	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4140	Assert(pCtx->gs.u32Limit == 0xffff);
4141	Assert(u32GSAttr == 0xf3);
4142	/* 64-bit capable CPUs. */
4143	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4144	Assert(!(pCtx->cs.u64Base >> 32));
4145	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4146	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4147	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4148	# endif
4149	}
4150	}
4151	#endif /* VBOX_STRICT */
4152
4153
4154	/**
4155	* Writes a guest segment register into the guest-state area in the VMCS.
4156	*
4157	* @returns VBox status code.
4158	* @param pVCpu Pointer to the VMCPU.
4159	* @param idxSel Index of the selector in the VMCS.
4160	* @param idxLimit Index of the segment limit in the VMCS.
4161	* @param idxBase Index of the segment base in the VMCS.
4162	* @param idxAccess Index of the access rights of the segment in the VMCS.
4163	* @param pSelReg Pointer to the segment selector.
4164	*
4165	* @remarks No-long-jump zone!!!
4166	*/
4167	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4168	uint32_t idxAccess, PCPUMSELREG pSelReg)
4169	{
4170	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4171	AssertRCReturn(rc, rc);
4172	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4173	AssertRCReturn(rc, rc);
4174	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4175	AssertRCReturn(rc, rc);
4176
4177	uint32_t u32Access = pSelReg->Attr.u;
4178	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4179	{
4180	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4181	u32Access = 0xf3;
4182	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4183	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4184	}
4185	else
4186	{
4187	/*
4188	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4189	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4190	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4191	* loaded in protected-mode have their attribute as 0.
4192	*/
4193	if (!u32Access)
4194	u32Access = X86DESCATTR_UNUSABLE;
4195	}
4196
4197	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4198	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4199	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4200
4201	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4202	AssertRCReturn(rc, rc);
4203	return rc;
4204	}
4205
4206
4207	/**
4208	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4209	* into the guest-state area in the VMCS.
4210	*
4211	* @returns VBox status code.
4212	* @param pVM Pointer to the VM.
4213	* @param pVCPU Pointer to the VMCPU.
4214	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4215	* out-of-sync. Make sure to update the required fields
4216	* before using them.
4217	*
4218	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4219	* @remarks No-long-jump zone!!!
4220	*/
4221	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4222	{
4223	int rc = VERR_INTERNAL_ERROR_5;
4224	PVM pVM = pVCpu->CTX_SUFF(pVM);
4225
4226	/*
4227	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4228	*/
4229	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4230	{
4231	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4232	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4233	{
4234	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4235	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4236	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4237	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4238	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4239	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4240	}
4241
4242	#ifdef VBOX_WITH_REM
4243	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4244	{
4245	Assert(pVM->hm.s.vmx.pRealModeTSS);
4246	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4247	if ( pVCpu->hm.s.vmx.fWasInRealMode
4248	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4249	{
4250	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4251	in real-mode (e.g. OpenBSD 4.0) */
4252	REMFlushTBs(pVM);
4253	Log4(("Load: Switch to protected mode detected!\n"));
4254	pVCpu->hm.s.vmx.fWasInRealMode = false;
4255	}
4256	}
4257	#endif
4258	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4259	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4260	AssertRCReturn(rc, rc);
4261	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4262	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4263	AssertRCReturn(rc, rc);
4264	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4265	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4266	AssertRCReturn(rc, rc);
4267	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4268	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4269	AssertRCReturn(rc, rc);
4270	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4271	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4272	AssertRCReturn(rc, rc);
4273	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4274	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4275	AssertRCReturn(rc, rc);
4276
4277	#ifdef VBOX_STRICT
4278	/* Validate. */
4279	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4280	#endif
4281
4282	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4283	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4284	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4285	}
4286
4287	/*
4288	* Guest TR.
4289	*/
4290	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4291	{
4292	/*
4293	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4294	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4295	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4296	*/
4297	uint16_t u16Sel = 0;
4298	uint32_t u32Limit = 0;
4299	uint64_t u64Base = 0;
4300	uint32_t u32AccessRights = 0;
4301
4302	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4303	{
4304	u16Sel = pMixedCtx->tr.Sel;
4305	u32Limit = pMixedCtx->tr.u32Limit;
4306	u64Base = pMixedCtx->tr.u64Base;
4307	u32AccessRights = pMixedCtx->tr.Attr.u;
4308	}
4309	else
4310	{
4311	Assert(pVM->hm.s.vmx.pRealModeTSS);
4312	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4313
4314	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4315	RTGCPHYS GCPhys;
4316	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4317	AssertRCReturn(rc, rc);
4318
4319	X86DESCATTR DescAttr;
4320	DescAttr.u = 0;
4321	DescAttr.n.u1Present = 1;
4322	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4323
4324	u16Sel = 0;
4325	u32Limit = HM_VTX_TSS_SIZE;
4326	u64Base = GCPhys; /* in real-mode phys = virt. */
4327	u32AccessRights = DescAttr.u;
4328	}
4329
4330	/* Validate. */
4331	Assert(!(u16Sel & RT_BIT(2)));
4332	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4333	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4334	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4335	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4336	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4337	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4338	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4339	Assert( (u32Limit & 0xfff) == 0xfff
4340	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4341	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4342	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4343
4344	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4345	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4346	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4347	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4348
4349	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4350	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4351	}
4352
4353	/*
4354	* Guest GDTR.
4355	*/
4356	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4357	{
4358	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4359	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4360
4361	/* Validate. */
4362	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4363
4364	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4365	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4366	}
4367
4368	/*
4369	* Guest LDTR.
4370	*/
4371	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4372	{
4373	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4374	uint32_t u32Access = 0;
4375	if (!pMixedCtx->ldtr.Attr.u)
4376	u32Access = X86DESCATTR_UNUSABLE;
4377	else
4378	u32Access = pMixedCtx->ldtr.Attr.u;
4379
4380	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4381	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4382	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4383	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4384
4385	/* Validate. */
4386	if (!(u32Access & X86DESCATTR_UNUSABLE))
4387	{
4388	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4389	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4390	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4391	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4392	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4393	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4394	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4395	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4396	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4397	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4398	}
4399
4400	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4401	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4402	}
4403
4404	/*
4405	* Guest IDTR.
4406	*/
4407	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4408	{
4409	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4410	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4411
4412	/* Validate. */
4413	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4414
4415	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4416	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4417	}
4418
4419	return VINF_SUCCESS;
4420	}
4421
4422
4423	/**
4424	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4425	* areas. These MSRs will automatically be loaded to the host CPU on every
4426	* successful VM entry and stored from the host CPU on every successful VM-exit.
4427	*
4428	* This also creates/updates MSR slots for the host MSRs. The actual host
4429	* MSR values are -not- updated here for performance reasons. See
4430	* hmR0VmxSaveHostMsrs().
4431	*
4432	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4433	*
4434	* @returns VBox status code.
4435	* @param pVCpu Pointer to the VMCPU.
4436	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4437	* out-of-sync. Make sure to update the required fields
4438	* before using them.
4439	*
4440	* @remarks No-long-jump zone!!!
4441	*/
4442	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4443	{
4444	AssertPtr(pVCpu);
4445	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4446
4447	/*
4448	* MSRs that we use the auto-load/store MSR area in the VMCS.
4449	*/
4450	PVM pVM = pVCpu->CTX_SUFF(pVM);
4451	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4452	{
4453	if (pVM->hm.s.fAllow64BitGuests)
4454	{
4455	#if HC_ARCH_BITS == 32 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4456	if (!HMVMX_IS_64BIT_HOST_MODE())
4457	{
4458	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4459	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4460	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4461	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4462	}
4463	# ifdef DEBUG
4464	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4465	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4466	Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4467	# endif
4468	#endif
4469	}
4470	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4471	}
4472
4473	/*
4474	* Guest Sysenter MSRs.
4475	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4476	* VM-exits on WRMSRs for these MSRs.
4477	*/
4478	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4479	{
4480	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4481	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4482	}
4483
4484	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4485	{
4486	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4487	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4488	}
4489
4490	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4491	{
4492	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4493	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4494	}
4495
4496	return VINF_SUCCESS;
4497	}
4498
4499
4500	/**
4501	* Loads the guest activity state into the guest-state area in the VMCS.
4502	*
4503	* @returns VBox status code.
4504	* @param pVCpu Pointer to the VMCPU.
4505	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4506	* out-of-sync. Make sure to update the required fields
4507	* before using them.
4508	*
4509	* @remarks No-long-jump zone!!!
4510	*/
4511	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4512	{
4513	NOREF(pCtx);
4514	/** @todo See if we can make use of other states, e.g.
4515	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4516	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4517	{
4518	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4519	AssertRCReturn(rc, rc);
4520
4521	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4522	}
4523	return VINF_SUCCESS;
4524	}
4525
4526
4527	/**
4528	* Sets up the appropriate function to run guest code.
4529	*
4530	* @returns VBox status code.
4531	* @param pVCpu Pointer to the VMCPU.
4532	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4533	* out-of-sync. Make sure to update the required fields
4534	* before using them.
4535	*
4536	* @remarks No-long-jump zone!!!
4537	*/
4538	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4539	{
4540	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4541	{
4542	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4543	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4544	#endif
4545	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4546	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4547	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4548	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4549	{
4550	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4551	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4552	}
4553	#else
4554	/* 64-bit host or hybrid host. */
4555	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4556	#endif
4557	}
4558	else
4559	{
4560	/* Guest is not in long mode, use the 32-bit handler. */
4561	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4562	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4563	{
4564	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4565	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4566	}
4567	#else
4568	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4569	#endif
4570	}
4571	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4572	return VINF_SUCCESS;
4573	}
4574
4575
4576	/**
4577	* Wrapper for running the guest code in VT-x.
4578	*
4579	* @returns VBox strict status code.
4580	* @param pVM Pointer to the VM.
4581	* @param pVCpu Pointer to the VMCPU.
4582	* @param pCtx Pointer to the guest-CPU context.
4583	*
4584	* @remarks No-long-jump zone!!!
4585	*/
4586	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4587	{
4588	/*
4589	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4590	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4591	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4592	*/
4593	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4594	/** @todo Add stats for resume vs launch. */
4595	#ifdef VBOX_WITH_KERNEL_USING_XMM
4596	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4597	#else
4598	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4599	#endif
4600	}
4601
4602
4603	/**
4604	* Reports world-switch error and dumps some useful debug info.
4605	*
4606	* @param pVM Pointer to the VM.
4607	* @param pVCpu Pointer to the VMCPU.
4608	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4609	* @param pCtx Pointer to the guest-CPU context.
4610	* @param pVmxTransient Pointer to the VMX transient structure (only
4611	* exitReason updated).
4612	*/
4613	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4614	{
4615	Assert(pVM);
4616	Assert(pVCpu);
4617	Assert(pCtx);
4618	Assert(pVmxTransient);
4619	HMVMX_ASSERT_PREEMPT_SAFE();
4620
4621	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4622	switch (rcVMRun)
4623	{
4624	case VERR_VMX_INVALID_VMXON_PTR:
4625	AssertFailed();
4626	break;
4627	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4628	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4629	{
4630	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4631	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4632	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4633	AssertRC(rc);
4634
4635	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4636	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4637	Cannot do it here as we may have been long preempted. */
4638
4639	#ifdef VBOX_STRICT
4640	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4641	pVmxTransient->uExitReason));
4642	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4643	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4644	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4645	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4646	else
4647	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4648	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4649	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4650
4651	/* VMX control bits. */
4652	uint32_t u32Val;
4653	uint64_t u64Val;
4654	HMVMXHCUINTREG uHCReg;
4655	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4656	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4657	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4658	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4659	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4660	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4661	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4662	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4663	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4664	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4665	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4666	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4667	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4668	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4669	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4670	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4671	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4672	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4673	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4674	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4675	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4676	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4677	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4678	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4679	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4680	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4681	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4682	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4683	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4684	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4685	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4686	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4687	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4688	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4689	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4690	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4691	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4692	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4693	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4694	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4695	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4696	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4697
4698	/* Guest bits. */
4699	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4700	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4701	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4702	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4703	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4704	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4705	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4706	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4707
4708	/* Host bits. */
4709	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4710	Log4(("Host CR0 %#RHr\n", uHCReg));
4711	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4712	Log4(("Host CR3 %#RHr\n", uHCReg));
4713	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4714	Log4(("Host CR4 %#RHr\n", uHCReg));
4715
4716	RTGDTR HostGdtr;
4717	PCX86DESCHC pDesc;
4718	ASMGetGDTR(&HostGdtr);
4719	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4720	Log4(("Host CS %#08x\n", u32Val));
4721	if (u32Val < HostGdtr.cbGdt)
4722	{
4723	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4724	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4725	}
4726
4727	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4728	Log4(("Host DS %#08x\n", u32Val));
4729	if (u32Val < HostGdtr.cbGdt)
4730	{
4731	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4732	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4733	}
4734
4735	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4736	Log4(("Host ES %#08x\n", u32Val));
4737	if (u32Val < HostGdtr.cbGdt)
4738	{
4739	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4740	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4741	}
4742
4743	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4744	Log4(("Host FS %#08x\n", u32Val));
4745	if (u32Val < HostGdtr.cbGdt)
4746	{
4747	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4748	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4749	}
4750
4751	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4752	Log4(("Host GS %#08x\n", u32Val));
4753	if (u32Val < HostGdtr.cbGdt)
4754	{
4755	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4756	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4757	}
4758
4759	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4760	Log4(("Host SS %#08x\n", u32Val));
4761	if (u32Val < HostGdtr.cbGdt)
4762	{
4763	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4764	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4765	}
4766
4767	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4768	Log4(("Host TR %#08x\n", u32Val));
4769	if (u32Val < HostGdtr.cbGdt)
4770	{
4771	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4772	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4773	}
4774
4775	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4776	Log4(("Host TR Base %#RHv\n", uHCReg));
4777	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4778	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4779	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4780	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4781	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4782	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4783	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4784	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4785	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4786	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4787	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4788	Log4(("Host RSP %#RHv\n", uHCReg));
4789	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4790	Log4(("Host RIP %#RHv\n", uHCReg));
4791	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4792	if (HMVMX_IS_64BIT_HOST_MODE())
4793	{
4794	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4795	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4796	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4797	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4798	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4799	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4800	}
4801	# endif
4802	#endif /* VBOX_STRICT */
4803	break;
4804	}
4805
4806	default:
4807	/* Impossible */
4808	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4809	break;
4810	}
4811	NOREF(pVM); NOREF(pCtx);
4812	}
4813
4814
4815	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4816	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4817	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4818	#endif
4819	#ifdef VBOX_STRICT
4820	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4821	{
4822	switch (idxField)
4823	{
4824	case VMX_VMCS_GUEST_RIP:
4825	case VMX_VMCS_GUEST_RSP:
4826	case VMX_VMCS_GUEST_SYSENTER_EIP:
4827	case VMX_VMCS_GUEST_SYSENTER_ESP:
4828	case VMX_VMCS_GUEST_GDTR_BASE:
4829	case VMX_VMCS_GUEST_IDTR_BASE:
4830	case VMX_VMCS_GUEST_CS_BASE:
4831	case VMX_VMCS_GUEST_DS_BASE:
4832	case VMX_VMCS_GUEST_ES_BASE:
4833	case VMX_VMCS_GUEST_FS_BASE:
4834	case VMX_VMCS_GUEST_GS_BASE:
4835	case VMX_VMCS_GUEST_SS_BASE:
4836	case VMX_VMCS_GUEST_LDTR_BASE:
4837	case VMX_VMCS_GUEST_TR_BASE:
4838	case VMX_VMCS_GUEST_CR3:
4839	return true;
4840	}
4841	return false;
4842	}
4843
4844	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4845	{
4846	switch (idxField)
4847	{
4848	/* Read-only fields. */
4849	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4850	return true;
4851	}
4852	/* Remaining readable fields should also be writable. */
4853	return hmR0VmxIsValidWriteField(idxField);
4854	}
4855	#endif /* VBOX_STRICT */
4856
4857
4858	/**
4859	* Executes the specified handler in 64-bit mode.
4860	*
4861	* @returns VBox status code.
4862	* @param pVM Pointer to the VM.
4863	* @param pVCpu Pointer to the VMCPU.
4864	* @param pCtx Pointer to the guest CPU context.
4865	* @param enmOp The operation to perform.
4866	* @param cbParam Number of parameters.
4867	* @param paParam Array of 32-bit parameters.
4868	*/
4869	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4870	uint32_t *paParam)
4871	{
4872	int rc, rc2;
4873	PHMGLOBALCPUINFO pCpu;
4874	RTHCPHYS HCPhysCpuPage;
4875	RTCCUINTREG uOldEflags;
4876
4877	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4878	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4879	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4880	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4881
4882	#ifdef VBOX_STRICT
4883	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4884	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4885
4886	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4887	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4888	#endif
4889
4890	/* Disable interrupts. */
4891	uOldEflags = ASMIntDisableFlags();
4892
4893	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4894	RTCPUID idHostCpu = RTMpCpuId();
4895	CPUMR0SetLApic(pVCpu, idHostCpu);
4896	#endif
4897
4898	pCpu = HMR0GetCurrentCpu();
4899	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4900
4901	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4902	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4903
4904	/* Leave VMX Root Mode. */
4905	VMXDisable();
4906
4907	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4908
4909	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4910	CPUMSetHyperEIP(pVCpu, enmOp);
4911	for (int i = (int)cbParam - 1; i >= 0; i--)
4912	CPUMPushHyper(pVCpu, paParam[i]);
4913
4914	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4915
4916	/* Call the switcher. */
4917	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4918	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4919
4920	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4921	/* Make sure the VMX instructions don't cause #UD faults. */
4922	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4923
4924	/* Re-enter VMX Root Mode */
4925	rc2 = VMXEnable(HCPhysCpuPage);
4926	if (RT_FAILURE(rc2))
4927	{
4928	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4929	ASMSetFlags(uOldEflags);
4930	return rc2;
4931	}
4932
4933	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4934	AssertRC(rc2);
4935	Assert(!(ASMGetFlags() & X86_EFL_IF));
4936	ASMSetFlags(uOldEflags);
4937	return rc;
4938	}
4939
4940
4941	/**
4942	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4943	* supporting 64-bit guests.
4944	*
4945	* @returns VBox status code.
4946	* @param fResume Whether to VMLAUNCH or VMRESUME.
4947	* @param pCtx Pointer to the guest-CPU context.
4948	* @param pCache Pointer to the VMCS cache.
4949	* @param pVM Pointer to the VM.
4950	* @param pVCpu Pointer to the VMCPU.
4951	*/
4952	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4953	{
4954	uint32_t aParam[6];
4955	PHMGLOBALCPUINFO pCpu = NULL;
4956	RTHCPHYS HCPhysCpuPage = 0;
4957	int rc = VERR_INTERNAL_ERROR_5;
4958
4959	pCpu = HMR0GetCurrentCpu();
4960	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4961
4962	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4963	pCache->uPos = 1;
4964	pCache->interPD = PGMGetInterPaeCR3(pVM);
4965	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4966	#endif
4967
4968	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4969	pCache->TestIn.HCPhysCpuPage = 0;
4970	pCache->TestIn.HCPhysVmcs = 0;
4971	pCache->TestIn.pCache = 0;
4972	pCache->TestOut.HCPhysVmcs = 0;
4973	pCache->TestOut.pCache = 0;
4974	pCache->TestOut.pCtx = 0;
4975	pCache->TestOut.eflags = 0;
4976	#endif
4977
4978	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4979	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4980	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4981	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4982	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4983	aParam[5] = 0;
4984
4985	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4986	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4987	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4988	#endif
4989	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4990
4991	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4992	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4993	Assert(pCtx->dr[4] == 10);
4994	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4995	#endif
4996
4997	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4998	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4999	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5000	pVCpu->hm.s.vmx.HCPhysVmcs));
5001	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5002	pCache->TestOut.HCPhysVmcs));
5003	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5004	pCache->TestOut.pCache));
5005	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5006	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5007	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5008	pCache->TestOut.pCtx));
5009	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5010	#endif
5011	return rc;
5012	}
5013
5014
5015	/**
5016	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5017	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5018	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5019	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5020	*
5021	* @returns VBox status code.
5022	* @param pVM Pointer to the VM.
5023	* @param pVCpu Pointer to the VMCPU.
5024	*/
5025	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5026	{
5027	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5028	{ \
5029	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5030	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5031	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5032	++cReadFields; \
5033	}
5034
5035	AssertPtr(pVM);
5036	AssertPtr(pVCpu);
5037	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5038	uint32_t cReadFields = 0;
5039
5040	/*
5041	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5042	* and serve to indicate exceptions to the rules.
5043	*/
5044
5045	/* Guest-natural selector base fields. */
5046	#if 0
5047	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5048	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5049	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5050	#endif
5051	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5052	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5053	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5054	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5055	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5056	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5057	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5058	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5059	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5060	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5061	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5062	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5063	#if 0
5064	/* Unused natural width guest-state fields. */
5065	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5066	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5067	#endif
5068	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5069	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5070
5071	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5072	#if 0
5073	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5074	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5075	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5076	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5077	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5078	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5079	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5080	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5081	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5082	#endif
5083
5084	/* Natural width guest-state fields. */
5085	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5086	#if 0
5087	/* Currently unused field. */
5088	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5089	#endif
5090
5091	if (pVM->hm.s.fNestedPaging)
5092	{
5093	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5094	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5095	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5096	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5097	}
5098	else
5099	{
5100	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5101	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5102	}
5103
5104	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5105	return VINF_SUCCESS;
5106	}
5107
5108
5109	/**
5110	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5111	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5112	* darwin, running 64-bit guests).
5113	*
5114	* @returns VBox status code.
5115	* @param pVCpu Pointer to the VMCPU.
5116	* @param idxField The VMCS field encoding.
5117	* @param u64Val 16, 32 or 64-bit value.
5118	*/
5119	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5120	{
5121	int rc;
5122	switch (idxField)
5123	{
5124	/*
5125	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5126	*/
5127	/* 64-bit Control fields. */
5128	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5129	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5130	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5131	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5132	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5133	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5134	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5135	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5136	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5137	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5138	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5139	case VMX_VMCS64_CTRL_EPTP_FULL:
5140	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5141	/* 64-bit Guest-state fields. */
5142	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5143	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5144	case VMX_VMCS64_GUEST_PAT_FULL:
5145	case VMX_VMCS64_GUEST_EFER_FULL:
5146	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5147	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5148	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5149	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5150	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5151	/* 64-bit Host-state fields. */
5152	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5153	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5154	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5155	{
5156	rc = VMXWriteVmcs32(idxField, u64Val);
5157	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5158	break;
5159	}
5160
5161	/*
5162	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5163	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5164	*/
5165	/* Natural-width Guest-state fields. */
5166	case VMX_VMCS_GUEST_CR3:
5167	case VMX_VMCS_GUEST_ES_BASE:
5168	case VMX_VMCS_GUEST_CS_BASE:
5169	case VMX_VMCS_GUEST_SS_BASE:
5170	case VMX_VMCS_GUEST_DS_BASE:
5171	case VMX_VMCS_GUEST_FS_BASE:
5172	case VMX_VMCS_GUEST_GS_BASE:
5173	case VMX_VMCS_GUEST_LDTR_BASE:
5174	case VMX_VMCS_GUEST_TR_BASE:
5175	case VMX_VMCS_GUEST_GDTR_BASE:
5176	case VMX_VMCS_GUEST_IDTR_BASE:
5177	case VMX_VMCS_GUEST_RSP:
5178	case VMX_VMCS_GUEST_RIP:
5179	case VMX_VMCS_GUEST_SYSENTER_ESP:
5180	case VMX_VMCS_GUEST_SYSENTER_EIP:
5181	{
5182	if (!(u64Val >> 32))
5183	{
5184	/* If this field is 64-bit, VT-x will zero out the top bits. */
5185	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5186	}
5187	else
5188	{
5189	/* Assert that only the 32->64 switcher case should ever come here. */
5190	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5191	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5192	}
5193	break;
5194	}
5195
5196	default:
5197	{
5198	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5199	rc = VERR_INVALID_PARAMETER;
5200	break;
5201	}
5202	}
5203	AssertRCReturn(rc, rc);
5204	return rc;
5205	}
5206
5207
5208	/**
5209	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5210	* hosts (except darwin) for 64-bit guests.
5211	*
5212	* @param pVCpu Pointer to the VMCPU.
5213	* @param idxField The VMCS field encoding.
5214	* @param u64Val 16, 32 or 64-bit value.
5215	*/
5216	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5217	{
5218	AssertPtr(pVCpu);
5219	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5220
5221	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5222	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5223
5224	/* Make sure there are no duplicates. */
5225	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5226	{
5227	if (pCache->Write.aField[i] == idxField)
5228	{
5229	pCache->Write.aFieldVal[i] = u64Val;
5230	return VINF_SUCCESS;
5231	}
5232	}
5233
5234	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5235	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5236	pCache->Write.cValidEntries++;
5237	return VINF_SUCCESS;
5238	}
5239
5240	/* Enable later when the assembly code uses these as callbacks. */
5241	#if 0
5242	/*
5243	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5244	*
5245	* @param pVCpu Pointer to the VMCPU.
5246	* @param pCache Pointer to the VMCS cache.
5247	*
5248	* @remarks No-long-jump zone!!!
5249	*/
5250	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5251	{
5252	AssertPtr(pCache);
5253	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5254	{
5255	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5256	AssertRC(rc);
5257	}
5258	pCache->Write.cValidEntries = 0;
5259	}
5260
5261
5262	/**
5263	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5264	*
5265	* @param pVCpu Pointer to the VMCPU.
5266	* @param pCache Pointer to the VMCS cache.
5267	*
5268	* @remarks No-long-jump zone!!!
5269	*/
5270	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5271	{
5272	AssertPtr(pCache);
5273	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5274	{
5275	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5276	AssertRC(rc);
5277	}
5278	}
5279	#endif
5280	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5281
5282
5283	/**
5284	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5285	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5286	* timer.
5287	*
5288	* @returns VBox status code.
5289	* @param pVCpu Pointer to the VMCPU.
5290	*
5291	* @remarks No-long-jump zone!!!
5292	*/
5293	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5294	{
5295	int rc = VERR_INTERNAL_ERROR_5;
5296	bool fOffsettedTsc = false;
5297	PVM pVM = pVCpu->CTX_SUFF(pVM);
5298	if (pVM->hm.s.vmx.fUsePreemptTimer)
5299	{
5300	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5301
5302	/* Make sure the returned values have sane upper and lower boundaries. */
5303	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5304	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5305	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5306	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5307
5308	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5309	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5310	}
5311	else
5312	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5313
5314	if (fOffsettedTsc)
5315	{
5316	uint64_t u64CurTSC = ASMReadTSC();
5317	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5318	{
5319	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5320	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5321
5322	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5323	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5324	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5325	}
5326	else
5327	{
5328	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5329	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5330	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5331	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5332	}
5333	}
5334	else
5335	{
5336	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5337	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5338	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5339	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5340	}
5341	}
5342
5343
5344	/**
5345	* Determines if an exception is a contributory exception. Contributory
5346	* exceptions are ones which can cause double-faults. Page-fault is
5347	* intentionally not included here as it's a conditional contributory exception.
5348	*
5349	* @returns true if the exception is contributory, false otherwise.
5350	* @param uVector The exception vector.
5351	*/
5352	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5353	{
5354	switch (uVector)
5355	{
5356	case X86_XCPT_GP:
5357	case X86_XCPT_SS:
5358	case X86_XCPT_NP:
5359	case X86_XCPT_TS:
5360	case X86_XCPT_DE:
5361	return true;
5362	default:
5363	break;
5364	}
5365	return false;
5366	}
5367
5368
5369	/**
5370	* Sets an event as a pending event to be injected into the guest.
5371	*
5372	* @param pVCpu Pointer to the VMCPU.
5373	* @param u32IntInfo The VM-entry interruption-information field.
5374	* @param cbInstr The VM-entry instruction length in bytes (for software
5375	* interrupts, exceptions and privileged software
5376	* exceptions).
5377	* @param u32ErrCode The VM-entry exception error code.
5378	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5379	* page-fault.
5380	*
5381	* @remarks Statistics counter assumes this is a guest event being injected or
5382	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5383	* always incremented.
5384	*/
5385	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5386	RTGCUINTPTR GCPtrFaultAddress)
5387	{
5388	Assert(!pVCpu->hm.s.Event.fPending);
5389	pVCpu->hm.s.Event.fPending = true;
5390	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5391	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5392	pVCpu->hm.s.Event.cbInstr = cbInstr;
5393	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5394
5395	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5396	}
5397
5398
5399	/**
5400	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5401	*
5402	* @param pVCpu Pointer to the VMCPU.
5403	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5404	* out-of-sync. Make sure to update the required fields
5405	* before using them.
5406	*/
5407	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5408	{
5409	NOREF(pMixedCtx);
5410	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5411	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5412	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5413	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5414	}
5415
5416
5417	/**
5418	* Handle a condition that occurred while delivering an event through the guest
5419	* IDT.
5420	*
5421	* @returns VBox status code (informational error codes included).
5422	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5423	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5424	* continue execution of the guest which will delivery the #DF.
5425	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5426	*
5427	* @param pVCpu Pointer to the VMCPU.
5428	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5429	* out-of-sync. Make sure to update the required fields
5430	* before using them.
5431	* @param pVmxTransient Pointer to the VMX transient structure.
5432	*
5433	* @remarks No-long-jump zone!!!
5434	*/
5435	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5436	{
5437	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5438	AssertRCReturn(rc, rc);
5439	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5440	{
5441	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5442	AssertRCReturn(rc, rc);
5443
5444	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5445	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5446	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5447
5448	typedef enum
5449	{
5450	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5451	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5452	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5453	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5454	} VMXREFLECTXCPT;
5455
5456	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5457	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5458	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5459	{
5460	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5461	{
5462	enmReflect = VMXREFLECTXCPT_XCPT;
5463	#ifdef VBOX_STRICT
5464	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5465	&& uExitVector == X86_XCPT_PF)
5466	{
5467	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5468	}
5469	#endif
5470	if ( uExitVector == X86_XCPT_PF
5471	&& uIdtVector == X86_XCPT_PF)
5472	{
5473	pVmxTransient->fVectoringPF = true;
5474	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5475	}
5476	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5477	&& hmR0VmxIsContributoryXcpt(uExitVector)
5478	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5479	\|\| uIdtVector == X86_XCPT_PF))
5480	{
5481	enmReflect = VMXREFLECTXCPT_DF;
5482	}
5483	else if (uIdtVector == X86_XCPT_DF)
5484	enmReflect = VMXREFLECTXCPT_TF;
5485	}
5486	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5487	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5488	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5489	{
5490	/*
5491	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5492	* (whatever they are) as they reoccur when restarting the instruction.
5493	*/
5494	enmReflect = VMXREFLECTXCPT_XCPT;
5495	}
5496	}
5497	else
5498	{
5499	/*
5500	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5501	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5502	* original exception to the guest after handling the VM-exit.
5503	*/
5504	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5505	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5506	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5507	{
5508	enmReflect = VMXREFLECTXCPT_XCPT;
5509	}
5510	}
5511
5512	switch (enmReflect)
5513	{
5514	case VMXREFLECTXCPT_XCPT:
5515	{
5516	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5517	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5518	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5519
5520	uint32_t u32ErrCode = 0;
5521	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5522	{
5523	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5524	AssertRCReturn(rc, rc);
5525	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5526	}
5527
5528	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5529	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5530	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5531	rc = VINF_SUCCESS;
5532	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5533	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5534
5535	break;
5536	}
5537
5538	case VMXREFLECTXCPT_DF:
5539	{
5540	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5541	rc = VINF_HM_DOUBLE_FAULT;
5542	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5543	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5544
5545	break;
5546	}
5547
5548	case VMXREFLECTXCPT_TF:
5549	{
5550	rc = VINF_EM_RESET;
5551	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5552	uExitVector));
5553	break;
5554	}
5555
5556	default:
5557	Assert(rc == VINF_SUCCESS);
5558	break;
5559	}
5560	}
5561	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5562	return rc;
5563	}
5564
5565
5566	/**
5567	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5568	*
5569	* @returns VBox status code.
5570	* @param pVCpu Pointer to the VMCPU.
5571	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5572	* out-of-sync. Make sure to update the required fields
5573	* before using them.
5574	*
5575	* @remarks No-long-jump zone!!!
5576	*/
5577	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5578	{
5579	NOREF(pMixedCtx);
5580
5581	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5582	{
5583	uint32_t uVal = 0;
5584	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5585	AssertRCReturn(rc, rc);
5586
5587	uint32_t uShadow = 0;
5588	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5589	AssertRCReturn(rc, rc);
5590
5591	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5592	CPUMSetGuestCR0(pVCpu, uVal);
5593	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5594	}
5595	return VINF_SUCCESS;
5596	}
5597
5598
5599	/**
5600	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5601	*
5602	* @returns VBox status code.
5603	* @param pVCpu Pointer to the VMCPU.
5604	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5605	* out-of-sync. Make sure to update the required fields
5606	* before using them.
5607	*
5608	* @remarks No-long-jump zone!!!
5609	*/
5610	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5611	{
5612	NOREF(pMixedCtx);
5613
5614	int rc = VINF_SUCCESS;
5615	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5616	{
5617	uint32_t uVal = 0;
5618	uint32_t uShadow = 0;
5619	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5620	AssertRCReturn(rc, rc);
5621	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5622	AssertRCReturn(rc, rc);
5623
5624	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5625	CPUMSetGuestCR4(pVCpu, uVal);
5626	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5627	}
5628	return rc;
5629	}
5630
5631
5632	/**
5633	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5634	*
5635	* @returns VBox status code.
5636	* @param pVCpu Pointer to the VMCPU.
5637	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5638	* out-of-sync. Make sure to update the required fields
5639	* before using them.
5640	*
5641	* @remarks No-long-jump zone!!!
5642	*/
5643	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5644	{
5645	int rc = VINF_SUCCESS;
5646	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5647	{
5648	uint64_t u64Val = 0;
5649	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5650	AssertRCReturn(rc, rc);
5651
5652	pMixedCtx->rip = u64Val;
5653	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5654	}
5655	return rc;
5656	}
5657
5658
5659	/**
5660	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5661	*
5662	* @returns VBox status code.
5663	* @param pVCpu Pointer to the VMCPU.
5664	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5665	* out-of-sync. Make sure to update the required fields
5666	* before using them.
5667	*
5668	* @remarks No-long-jump zone!!!
5669	*/
5670	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5671	{
5672	int rc = VINF_SUCCESS;
5673	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5674	{
5675	uint64_t u64Val = 0;
5676	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5677	AssertRCReturn(rc, rc);
5678
5679	pMixedCtx->rsp = u64Val;
5680	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5681	}
5682	return rc;
5683	}
5684
5685
5686	/**
5687	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5688	*
5689	* @returns VBox status code.
5690	* @param pVCpu Pointer to the VMCPU.
5691	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5692	* out-of-sync. Make sure to update the required fields
5693	* before using them.
5694	*
5695	* @remarks No-long-jump zone!!!
5696	*/
5697	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5698	{
5699	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5700	{
5701	uint32_t uVal = 0;
5702	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5703	AssertRCReturn(rc, rc);
5704
5705	pMixedCtx->eflags.u32 = uVal;
5706	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5707	{
5708	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5709	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5710
5711	pMixedCtx->eflags.Bits.u1VM = 0;
5712	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5713	}
5714
5715	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5716	}
5717	return VINF_SUCCESS;
5718	}
5719
5720
5721	/**
5722	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5723	* guest-CPU context.
5724	*/
5725	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5726	{
5727	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5728	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5729	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5730	return rc;
5731	}
5732
5733
5734	/**
5735	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5736	* from the guest-state area in the VMCS.
5737	*
5738	* @param pVCpu Pointer to the VMCPU.
5739	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5740	* out-of-sync. Make sure to update the required fields
5741	* before using them.
5742	*
5743	* @remarks No-long-jump zone!!!
5744	*/
5745	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5746	{
5747	uint32_t uIntrState = 0;
5748	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5749	AssertRC(rc);
5750
5751	if (!uIntrState)
5752	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5753	else
5754	{
5755	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5756	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5757	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5758	AssertRC(rc);
5759	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5760	AssertRC(rc);
5761
5762	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5763	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5764	}
5765	}
5766
5767
5768	/**
5769	* Saves the guest's activity state.
5770	*
5771	* @returns VBox status code.
5772	* @param pVCpu Pointer to the VMCPU.
5773	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5774	* out-of-sync. Make sure to update the required fields
5775	* before using them.
5776	*
5777	* @remarks No-long-jump zone!!!
5778	*/
5779	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5780	{
5781	NOREF(pMixedCtx);
5782	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5783	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5784	return VINF_SUCCESS;
5785	}
5786
5787
5788	/**
5789	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5790	* the current VMCS into the guest-CPU context.
5791	*
5792	* @returns VBox status code.
5793	* @param pVCpu Pointer to the VMCPU.
5794	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5795	* out-of-sync. Make sure to update the required fields
5796	* before using them.
5797	*
5798	* @remarks No-long-jump zone!!!
5799	*/
5800	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5801	{
5802	int rc = VINF_SUCCESS;
5803	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5804	{
5805	uint32_t u32Val = 0;
5806	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5807	pMixedCtx->SysEnter.cs = u32Val;
5808	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5809	}
5810
5811	uint64_t u64Val = 0;
5812	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5813	{
5814	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5815	pMixedCtx->SysEnter.eip = u64Val;
5816	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5817	}
5818	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5819	{
5820	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5821	pMixedCtx->SysEnter.esp = u64Val;
5822	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5823	}
5824	return rc;
5825	}
5826
5827
5828	/**
5829	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5830	* the CPU back into the guest-CPU context.
5831	*
5832	* @returns VBox status code.
5833	* @param pVCpu Pointer to the VMCPU.
5834	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5835	* out-of-sync. Make sure to update the required fields
5836	* before using them.
5837	*
5838	* @remarks No-long-jump zone!!!
5839	*/
5840	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5841	{
5842	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5843	if ( HMVMX_IS_64BIT_HOST_MODE()
5844	&& pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5845	{
5846	/* We should not get preempted to a different CPU at this point while reading the MSRs. */
5847	VMMRZCallRing3Disable(pVCpu);
5848	HM_DISABLE_PREEMPT_IF_NEEDED();
5849
5850	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5851	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LAZY_MSRS))
5852	{
5853	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5854	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LAZY_MSRS;
5855	}
5856
5857	HM_RESTORE_PREEMPT_IF_NEEDED();
5858	VMMRZCallRing3Enable(pVCpu);
5859	}
5860	else
5861	{
5862	/* Darwin 32-bit/PAE kernel or 64-bit host running 32-bit guest. */
5863	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LAZY_MSRS;
5864	}
5865	#else /* HC_ARCH_BITS == 32 */
5866	NOREF(pMixedCtx);
5867	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LAZY_MSRS;
5868	#endif
5869
5870	return VINF_SUCCESS;
5871	}
5872
5873
5874	/**
5875	* Saves the auto load/store'd guest MSRs from the current VMCS into
5876	* the guest-CPU context.
5877	*
5878	* @returns VBox status code.
5879	* @param pVCpu Pointer to the VMCPU.
5880	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5881	* out-of-sync. Make sure to update the required fields
5882	* before using them.
5883	*
5884	* @remarks No-long-jump zone!!!
5885	*/
5886	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5887	{
5888	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5889	return VINF_SUCCESS;
5890
5891	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5892	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5893	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5894	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5895	{
5896	switch (pMsr->u32Msr)
5897	{
5898	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5899	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5900	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5901	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5902	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5903	default:
5904	{
5905	AssertFailed();
5906	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5907	}
5908	}
5909	}
5910
5911	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5912	return VINF_SUCCESS;
5913	}
5914
5915
5916	/**
5917	* Saves the guest control registers from the current VMCS into the guest-CPU
5918	* context.
5919	*
5920	* @returns VBox status code.
5921	* @param pVCpu Pointer to the VMCPU.
5922	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5923	* out-of-sync. Make sure to update the required fields
5924	* before using them.
5925	*
5926	* @remarks No-long-jump zone!!!
5927	*/
5928	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5929	{
5930	/* Guest CR0. Guest FPU. */
5931	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5932	AssertRCReturn(rc, rc);
5933
5934	/* Guest CR4. */
5935	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5936	AssertRCReturn(rc, rc);
5937
5938	/* Guest CR2 - updated always during the world-switch or in #PF. */
5939	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5940	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5941	{
5942	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5943	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5944
5945	PVM pVM = pVCpu->CTX_SUFF(pVM);
5946	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5947	\|\| ( pVM->hm.s.fNestedPaging
5948	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5949	{
5950	uint64_t u64Val = 0;
5951	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5952	if (pMixedCtx->cr3 != u64Val)
5953	{
5954	CPUMSetGuestCR3(pVCpu, u64Val);
5955	if (VMMRZCallRing3IsEnabled(pVCpu))
5956	{
5957	PGMUpdateCR3(pVCpu, u64Val);
5958	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5959	}
5960	else
5961	{
5962	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5963	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5964	}
5965	}
5966
5967	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5968	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5969	{
5970	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5971	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5972	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5973	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5974
5975	if (VMMRZCallRing3IsEnabled(pVCpu))
5976	{
5977	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5978	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5979	}
5980	else
5981	{
5982	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5983	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5984	}
5985	}
5986	}
5987
5988	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5989	}
5990
5991	/*
5992	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5993	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5994	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5995	*
5996	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5997	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5998	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5999	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6000	*
6001	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6002	*/
6003	if (VMMRZCallRing3IsEnabled(pVCpu))
6004	{
6005	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6006	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6007
6008	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6009	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6010
6011	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6012	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6013	}
6014
6015	return rc;
6016	}
6017
6018
6019	/**
6020	* Reads a guest segment register from the current VMCS into the guest-CPU
6021	* context.
6022	*
6023	* @returns VBox status code.
6024	* @param pVCpu Pointer to the VMCPU.
6025	* @param idxSel Index of the selector in the VMCS.
6026	* @param idxLimit Index of the segment limit in the VMCS.
6027	* @param idxBase Index of the segment base in the VMCS.
6028	* @param idxAccess Index of the access rights of the segment in the VMCS.
6029	* @param pSelReg Pointer to the segment selector.
6030	*
6031	* @remarks No-long-jump zone!!!
6032	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6033	* macro as that takes care of whether to read from the VMCS cache or
6034	* not.
6035	*/
6036	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6037	PCPUMSELREG pSelReg)
6038	{
6039	NOREF(pVCpu);
6040
6041	uint32_t u32Val = 0;
6042	int rc = VMXReadVmcs32(idxSel, &u32Val);
6043	AssertRCReturn(rc, rc);
6044	pSelReg->Sel = (uint16_t)u32Val;
6045	pSelReg->ValidSel = (uint16_t)u32Val;
6046	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6047
6048	rc = VMXReadVmcs32(idxLimit, &u32Val);
6049	AssertRCReturn(rc, rc);
6050	pSelReg->u32Limit = u32Val;
6051
6052	uint64_t u64Val = 0;
6053	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6054	AssertRCReturn(rc, rc);
6055	pSelReg->u64Base = u64Val;
6056
6057	rc = VMXReadVmcs32(idxAccess, &u32Val);
6058	AssertRCReturn(rc, rc);
6059	pSelReg->Attr.u = u32Val;
6060
6061	/*
6062	* If VT-x marks the segment as unusable, most other bits remain undefined:
6063	* - For CS the L, D and G bits have meaning.
6064	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6065	* - For the remaining data segments no bits are defined.
6066	*
6067	* The present bit and the unusable bit has been observed to be set at the
6068	* same time (the selector was supposed to invalid as we started executing
6069	* a V8086 interrupt in ring-0).
6070	*
6071	* What should be important for the rest of the VBox code that the P bit is
6072	* cleared. Some of the other VBox code recognizes the unusable bit, but
6073	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6074	* safe side here, we'll strip off P and other bits we don't care about. If
6075	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6076	*
6077	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6078	*/
6079	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6080	{
6081	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6082
6083	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6084	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6085	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6086
6087	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6088	#ifdef DEBUG_bird
6089	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6090	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6091	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6092	#endif
6093	}
6094	return VINF_SUCCESS;
6095	}
6096
6097
6098	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6099	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6100	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6101	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6102	#else
6103	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6104	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6105	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6106	#endif
6107
6108
6109	/**
6110	* Saves the guest segment registers from the current VMCS into the guest-CPU
6111	* context.
6112	*
6113	* @returns VBox status code.
6114	* @param pVCpu Pointer to the VMCPU.
6115	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6116	* out-of-sync. Make sure to update the required fields
6117	* before using them.
6118	*
6119	* @remarks No-long-jump zone!!!
6120	*/
6121	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6122	{
6123	/* Guest segment registers. */
6124	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6125	{
6126	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6127	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6128	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6129	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6130	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6131	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6132	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6133
6134	/* Restore segment attributes for real-on-v86 mode hack. */
6135	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6136	{
6137	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6138	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6139	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6140	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6141	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6142	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6143	}
6144	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
6145	}
6146
6147	return VINF_SUCCESS;
6148	}
6149
6150
6151	/**
6152	* Saves the guest descriptor table registers and task register from the current
6153	* VMCS into the guest-CPU context.
6154	*
6155	* @returns VBox status code.
6156	* @param pVCpu Pointer to the VMCPU.
6157	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6158	* out-of-sync. Make sure to update the required fields
6159	* before using them.
6160	*
6161	* @remarks No-long-jump zone!!!
6162	*/
6163	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6164	{
6165	int rc = VINF_SUCCESS;
6166
6167	/* Guest LDTR. */
6168	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
6169	{
6170	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6171	AssertRCReturn(rc, rc);
6172	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
6173	}
6174
6175	/* Guest GDTR. */
6176	uint64_t u64Val = 0;
6177	uint32_t u32Val = 0;
6178	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
6179	{
6180	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6181	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6182	pMixedCtx->gdtr.pGdt = u64Val;
6183	pMixedCtx->gdtr.cbGdt = u32Val;
6184	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
6185	}
6186
6187	/* Guest IDTR. */
6188	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
6189	{
6190	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6191	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6192	pMixedCtx->idtr.pIdt = u64Val;
6193	pMixedCtx->idtr.cbIdt = u32Val;
6194	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
6195	}
6196
6197	/* Guest TR. */
6198	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
6199	{
6200	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6201	AssertRCReturn(rc, rc);
6202
6203	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6204	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6205	{
6206	rc = VMXLOCAL_READ_SEG(TR, tr);
6207	AssertRCReturn(rc, rc);
6208	}
6209	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
6210	}
6211	return rc;
6212	}
6213
6214	#undef VMXLOCAL_READ_SEG
6215
6216
6217	/**
6218	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6219	* context.
6220	*
6221	* @returns VBox status code.
6222	* @param pVCpu Pointer to the VMCPU.
6223	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6224	* out-of-sync. Make sure to update the required fields
6225	* before using them.
6226	*
6227	* @remarks No-long-jump zone!!!
6228	*/
6229	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6230	{
6231	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
6232	{
6233	if (!pVCpu->hm.s.fUsingHyperDR7)
6234	{
6235	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6236	uint32_t u32Val;
6237	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6238	pMixedCtx->dr[7] = u32Val;
6239	}
6240
6241	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
6242	}
6243	return VINF_SUCCESS;
6244	}
6245
6246
6247	/**
6248	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6249	*
6250	* @returns VBox status code.
6251	* @param pVCpu Pointer to the VMCPU.
6252	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6253	* out-of-sync. Make sure to update the required fields
6254	* before using them.
6255	*
6256	* @remarks No-long-jump zone!!!
6257	*/
6258	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6259	{
6260	NOREF(pMixedCtx);
6261
6262	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6263	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
6264	return VINF_SUCCESS;
6265	}
6266
6267
6268	/**
6269	* Saves the entire guest state from the currently active VMCS into the
6270	* guest-CPU context. This essentially VMREADs all guest-data.
6271	*
6272	* @returns VBox status code.
6273	* @param pVCpu Pointer to the VMCPU.
6274	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6275	* out-of-sync. Make sure to update the required fields
6276	* before using them.
6277	*/
6278	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6279	{
6280	Assert(pVCpu);
6281	Assert(pMixedCtx);
6282
6283	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
6284	return VINF_SUCCESS;
6285
6286	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6287	again on the ring-3 callback path, there is no real need to. */
6288	if (VMMRZCallRing3IsEnabled(pVCpu))
6289	VMMR0LogFlushDisable(pVCpu);
6290	else
6291	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6292	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6293
6294	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6295	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6296
6297	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6298	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6299
6300	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6301	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6302
6303	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6304	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6305
6306	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6307	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6308
6309	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6310	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6311
6312	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6313	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6314
6315	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6317
6318	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6320
6321	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6323
6324	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
6325	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
6326
6327	if (VMMRZCallRing3IsEnabled(pVCpu))
6328	VMMR0LogFlushEnable(pVCpu);
6329
6330	return rc;
6331	}
6332
6333
6334	/**
6335	* Check per-VM and per-VCPU force flag actions that require us to go back to
6336	* ring-3 for one reason or another.
6337	*
6338	* @returns VBox status code (information status code included).
6339	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6340	* ring-3.
6341	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6342	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6343	* interrupts)
6344	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6345	* all EMTs to be in ring-3.
6346	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6347	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6348	* to the EM loop.
6349	*
6350	* @param pVM Pointer to the VM.
6351	* @param pVCpu Pointer to the VMCPU.
6352	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6353	* out-of-sync. Make sure to update the required fields
6354	* before using them.
6355	*/
6356	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6357	{
6358	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6359
6360	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6361	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6362	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6363	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6364	{
6365	/* We need the control registers now, make sure the guest-CPU context is updated. */
6366	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6367	AssertRCReturn(rc3, rc3);
6368
6369	/* Pending HM CR3 sync. */
6370	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6371	{
6372	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6373	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6374	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6375	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6376	}
6377
6378	/* Pending HM PAE PDPEs. */
6379	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6380	{
6381	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6382	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6383	}
6384
6385	/* Pending PGM C3 sync. */
6386	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6387	{
6388	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6389	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6390	if (rc2 != VINF_SUCCESS)
6391	{
6392	AssertRC(rc2);
6393	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6394	return rc2;
6395	}
6396	}
6397
6398	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6399	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6400	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6401	{
6402	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6403	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6404	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6405	return rc2;
6406	}
6407
6408	/* Pending VM request packets, such as hardware interrupts. */
6409	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6410	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6411	{
6412	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6413	return VINF_EM_PENDING_REQUEST;
6414	}
6415
6416	/* Pending PGM pool flushes. */
6417	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6418	{
6419	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6420	return VINF_PGM_POOL_FLUSH_PENDING;
6421	}
6422
6423	/* Pending DMA requests. */
6424	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6425	{
6426	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6427	return VINF_EM_RAW_TO_R3;
6428	}
6429	}
6430
6431	return VINF_SUCCESS;
6432	}
6433
6434
6435	/**
6436	* Converts any TRPM trap into a pending HM event. This is typically used when
6437	* entering from ring-3 (not longjmp returns).
6438	*
6439	* @param pVCpu Pointer to the VMCPU.
6440	*/
6441	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6442	{
6443	Assert(TRPMHasTrap(pVCpu));
6444	Assert(!pVCpu->hm.s.Event.fPending);
6445
6446	uint8_t uVector;
6447	TRPMEVENT enmTrpmEvent;
6448	RTGCUINT uErrCode;
6449	RTGCUINTPTR GCPtrFaultAddress;
6450	uint8_t cbInstr;
6451
6452	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6453	AssertRC(rc);
6454
6455	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6456	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6457	if (enmTrpmEvent == TRPM_TRAP)
6458	{
6459	switch (uVector)
6460	{
6461	case X86_XCPT_BP:
6462	case X86_XCPT_OF:
6463	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6464	break;
6465
6466	case X86_XCPT_PF:
6467	case X86_XCPT_DF:
6468	case X86_XCPT_TS:
6469	case X86_XCPT_NP:
6470	case X86_XCPT_SS:
6471	case X86_XCPT_GP:
6472	case X86_XCPT_AC:
6473	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6474	/* no break! */
6475	default:
6476	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6477	break;
6478	}
6479	}
6480	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6481	{
6482	if (uVector == X86_XCPT_NMI)
6483	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6484	else
6485	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6486	}
6487	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6488	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6489	else
6490	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6491
6492	rc = TRPMResetTrap(pVCpu);
6493	AssertRC(rc);
6494	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6495	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6496
6497	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6498	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6499	}
6500
6501
6502	/**
6503	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6504	* VT-x to execute any instruction.
6505	*
6506	* @param pvCpu Pointer to the VMCPU.
6507	*/
6508	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6509	{
6510	Assert(pVCpu->hm.s.Event.fPending);
6511
6512	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6513	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6514	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6515	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6516
6517	/* If a trap was already pending, we did something wrong! */
6518	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6519
6520	TRPMEVENT enmTrapType;
6521	switch (uVectorType)
6522	{
6523	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6524	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6525	enmTrapType = TRPM_HARDWARE_INT;
6526	break;
6527
6528	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6529	enmTrapType = TRPM_SOFTWARE_INT;
6530	break;
6531
6532	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6533	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6534	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6535	enmTrapType = TRPM_TRAP;
6536	break;
6537
6538	default:
6539	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6540	enmTrapType = TRPM_32BIT_HACK;
6541	break;
6542	}
6543
6544	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6545
6546	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6547	AssertRC(rc);
6548
6549	if (fErrorCodeValid)
6550	TRPMSetErrorCode(pVCpu, uErrorCode);
6551
6552	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6553	&& uVector == X86_XCPT_PF)
6554	{
6555	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6556	}
6557	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6558	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6559	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6560	{
6561	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6562	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6563	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6564	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6565	}
6566	pVCpu->hm.s.Event.fPending = false;
6567	}
6568
6569
6570	/**
6571	* Does the necessary state syncing before returning to ring-3 for any reason
6572	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6573	*
6574	* @returns VBox status code.
6575	* @param pVM Pointer to the VM.
6576	* @param pVCpu Pointer to the VMCPU.
6577	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6578	* be out-of-sync. Make sure to update the required
6579	* fields before using them.
6580	* @param fSaveGuestState Whether to save the guest state or not.
6581	*
6582	* @remarks If you modify code here, make sure to check whether
6583	* hmR0VmxCallRing3Callback() needs to be updated too!!!
6584	* @remarks No-long-jmp zone!!!
6585	*/
6586	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6587	{
6588	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6589	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6590
6591	RTCPUID idCpu = RTMpCpuId();
6592	Log4Func(("HostCpuId=%u\n", idCpu));
6593
6594	/* Save the guest state if necessary. */
6595	if ( fSaveGuestState
6596	&& pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6597	{
6598	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6599	AssertRCReturn(rc, rc);
6600	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6601	}
6602
6603	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6604	if (CPUMIsGuestFPUStateActive(pVCpu))
6605	{
6606	/* We shouldn't reload CR0 without saving it first. */
6607	if (!fSaveGuestState)
6608	{
6609	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6610	AssertRCReturn(rc, rc);
6611	}
6612	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6613	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6614	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6615	}
6616
6617	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6618	#ifdef VBOX_STRICT
6619	if (CPUMIsHyperDebugStateActive(pVCpu))
6620	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6621	#endif
6622	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6623	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6624	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6625	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6626
6627	#if HC_ARCH_BITS == 64
6628	/* Restore host-state bits that VT-x only restores partially. */
6629	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6630	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6631	{
6632	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6633	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6634	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6635	}
6636	#endif
6637
6638	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
6639	/* Restore the host MSRs as we're leaving VT-x context. */
6640	if ( HMVMX_IS_64BIT_HOST_MODE()
6641	&& pVM->hm.s.fAllow64BitGuests
6642	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6643	{
6644	/* We shouldn't reload the guest MSRs without saving it first. */
6645	if (!fSaveGuestState)
6646	{
6647	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6648	AssertRCReturn(rc, rc);
6649	}
6650	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LAZY_MSRS);
6651	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6652	Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6653	}
6654	#endif
6655
6656	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6657	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6658	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6659	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6660	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6661	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6662	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6663	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6664
6665	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6666
6667	/** @todo This partially defeats the purpose of having preemption hooks.
6668	* The problem is, deregistering the hooks should be moved to a place that
6669	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6670	* context.
6671	*/
6672	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6673	{
6674	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6675	AssertRCReturn(rc, rc);
6676
6677	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6678	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6679	}
6680	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6681	NOREF(idCpu);
6682
6683	return VINF_SUCCESS;
6684	}
6685
6686
6687	/**
6688	* Leaves the VT-x session.
6689	*
6690	* @returns VBox status code.
6691	* @param pVM Pointer to the VM.
6692	* @param pVCpu Pointer to the VMCPU.
6693	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6694	* out-of-sync. Make sure to update the required fields
6695	* before using them.
6696	*
6697	* @remarks No-long-jmp zone!!!
6698	*/
6699	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6700	{
6701	HM_DISABLE_PREEMPT_IF_NEEDED();
6702	HMVMX_ASSERT_CPU_SAFE();
6703	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6704	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6705
6706	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6707	and done this from the VMXR0ThreadCtxCallback(). */
6708	if (!pVCpu->hm.s.fLeaveDone)
6709	{
6710	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6711	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6712	pVCpu->hm.s.fLeaveDone = true;
6713	}
6714
6715	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6716	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6717	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6718	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6719	VMMR0ThreadCtxHooksDeregister(pVCpu);
6720
6721	/* Leave HM context. This takes care of local init (term). */
6722	int rc = HMR0LeaveCpu(pVCpu);
6723
6724	HM_RESTORE_PREEMPT_IF_NEEDED();
6725
6726	return rc;
6727	}
6728
6729
6730	/**
6731	* Does the necessary state syncing before doing a longjmp to ring-3.
6732	*
6733	* @returns VBox status code.
6734	* @param pVM Pointer to the VM.
6735	* @param pVCpu Pointer to the VMCPU.
6736	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6737	* out-of-sync. Make sure to update the required fields
6738	* before using them.
6739	*
6740	* @remarks No-long-jmp zone!!!
6741	*/
6742	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6743	{
6744	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6745	}
6746
6747
6748	/**
6749	* Take necessary actions before going back to ring-3.
6750	*
6751	* An action requires us to go back to ring-3. This function does the necessary
6752	* steps before we can safely return to ring-3. This is not the same as longjmps
6753	* to ring-3, this is voluntary and prepares the guest so it may continue
6754	* executing outside HM (recompiler/IEM).
6755	*
6756	* @returns VBox status code.
6757	* @param pVM Pointer to the VM.
6758	* @param pVCpu Pointer to the VMCPU.
6759	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6760	* out-of-sync. Make sure to update the required fields
6761	* before using them.
6762	* @param rcExit The reason for exiting to ring-3. Can be
6763	* VINF_VMM_UNKNOWN_RING3_CALL.
6764	*/
6765	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6766	{
6767	Assert(pVM);
6768	Assert(pVCpu);
6769	Assert(pMixedCtx);
6770	HMVMX_ASSERT_PREEMPT_SAFE();
6771
6772	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6773	{
6774	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6775	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6776	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6777	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6778	}
6779
6780	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6781	VMMRZCallRing3Disable(pVCpu);
6782	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6783
6784	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6785	if (pVCpu->hm.s.Event.fPending)
6786	{
6787	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6788	Assert(!pVCpu->hm.s.Event.fPending);
6789	}
6790
6791	/* Save guest state and restore host state bits. */
6792	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6793	AssertRCReturn(rc, rc);
6794	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6795
6796	/* Sync recompiler state. */
6797	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6798	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6799	\| CPUM_CHANGED_LDTR
6800	\| CPUM_CHANGED_GDTR
6801	\| CPUM_CHANGED_IDTR
6802	\| CPUM_CHANGED_TR
6803	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6804	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6805	if ( pVM->hm.s.fNestedPaging
6806	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6807	{
6808	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6809	}
6810
6811	Assert(!pVCpu->hm.s.fClearTrapFlag);
6812
6813	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6814	if (rcExit != VINF_EM_RAW_INTERRUPT)
6815	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6816
6817	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6818
6819	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6820	VMMRZCallRing3RemoveNotification(pVCpu);
6821	VMMRZCallRing3Enable(pVCpu);
6822
6823	return rc;
6824	}
6825
6826
6827	/**
6828	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6829	* longjump to ring-3 and possibly get preempted.
6830	*
6831	* @returns VBox status code.
6832	* @param pVCpu Pointer to the VMCPU.
6833	* @param enmOperation The operation causing the ring-3 longjump.
6834	* @param pvUser Opaque pointer to the guest-CPU context. The data
6835	* may be out-of-sync. Make sure to update the required
6836	* fields before using them.
6837	*
6838	* @remarks If you modify code here, make sure to check whether
6839	* hmR0VmxLeave() needs to be updated too!!!
6840	*/
6841	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6842	{
6843	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6844	{
6845	/* If anything here asserts or fails, good luck. */
6846	VMMRZCallRing3RemoveNotification(pVCpu);
6847	VMMRZCallRing3Disable(pVCpu);
6848	HM_DISABLE_PREEMPT_IF_NEEDED();
6849
6850	PVM pVM = pVCpu->CTX_SUFF(pVM);
6851	if (CPUMIsGuestFPUStateActive(pVCpu))
6852	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6853
6854	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6855
6856	#if HC_ARCH_BITS == 64
6857	/* Restore host-state bits that VT-x only restores partially. */
6858	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6859	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6860	{
6861	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6862	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6863	}
6864	#endif
6865
6866	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
6867	/* Restore the host MSRs as we're leaving VT-x context. */
6868	if ( HMVMX_IS_64BIT_HOST_MODE()
6869	&& pVM->hm.s.fAllow64BitGuests
6870	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6871	{
6872	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6873	}
6874	#endif
6875	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6876	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6877	{
6878	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6879	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6880	}
6881
6882	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6883	VMMR0ThreadCtxHooksDeregister(pVCpu);
6884
6885	HMR0LeaveCpu(pVCpu);
6886	HM_RESTORE_PREEMPT_IF_NEEDED();
6887	return VINF_SUCCESS;
6888	}
6889
6890	Assert(pVCpu);
6891	Assert(pvUser);
6892	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6893	HMVMX_ASSERT_PREEMPT_SAFE();
6894
6895	VMMRZCallRing3Disable(pVCpu);
6896	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6897
6898	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6899	enmOperation));
6900
6901	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6902	AssertRCReturn(rc, rc);
6903
6904	VMMRZCallRing3Enable(pVCpu);
6905	return VINF_SUCCESS;
6906	}
6907
6908
6909	/**
6910	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6911	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6912	*
6913	* @param pVCpu Pointer to the VMCPU.
6914	*/
6915	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6916	{
6917	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6918	{
6919	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6920	{
6921	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6922	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6923	AssertRC(rc);
6924	}
6925	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6926	}
6927
6928
6929	/**
6930	* Evaluates the event to be delivered to the guest and sets it as the pending
6931	* event.
6932	*
6933	* @param pVCpu Pointer to the VMCPU.
6934	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6935	* out-of-sync. Make sure to update the required fields
6936	* before using them.
6937	*/
6938	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6939	{
6940	Assert(!pVCpu->hm.s.Event.fPending);
6941
6942	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6943	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6944	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6945	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6946
6947	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6948	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6949	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6950	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6951	Assert(!TRPMHasTrap(pVCpu));
6952
6953	/** @todo SMI. SMIs take priority over NMIs. */
6954	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6955	{
6956	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6957	if ( !fBlockMovSS
6958	&& !fBlockSti)
6959	{
6960	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6961	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6962	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6963	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6964
6965	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6966	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6967	}
6968	else
6969	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6970	}
6971	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6972	&& !pVCpu->hm.s.fSingleInstruction)
6973	{
6974	/*
6975	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6976	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6977	* evaluated here and not set as pending, solely based on the force-flags.
6978	*/
6979	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6980	AssertRC(rc);
6981	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6982	if ( !fBlockInt
6983	&& !fBlockSti
6984	&& !fBlockMovSS)
6985	{
6986	uint8_t u8Interrupt;
6987	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6988	if (RT_SUCCESS(rc))
6989	{
6990	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6991	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6992	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6993
6994	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
6995	}
6996	else
6997	{
6998	/** @todo Does this actually happen? If not turn it into an assertion. */
6999	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7000	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7001	}
7002	}
7003	else
7004	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7005	}
7006	}
7007
7008
7009	/**
7010	* Injects any pending events into the guest if the guest is in a state to
7011	* receive them.
7012	*
7013	* @returns VBox status code (informational status codes included).
7014	* @param pVCpu Pointer to the VMCPU.
7015	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7016	* out-of-sync. Make sure to update the required fields
7017	* before using them.
7018	*/
7019	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7020	{
7021	HMVMX_ASSERT_PREEMPT_SAFE();
7022	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7023
7024	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7025	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7026	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7027	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7028
7029	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
7030	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7031	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7032	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7033	Assert(!TRPMHasTrap(pVCpu));
7034
7035	int rc = VINF_SUCCESS;
7036	if (pVCpu->hm.s.Event.fPending)
7037	{
7038	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
7039	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7040	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7041	{
7042	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7043	AssertRCReturn(rc, rc);
7044	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7045	Assert(!fBlockInt);
7046	Assert(!fBlockSti);
7047	Assert(!fBlockMovSS);
7048	}
7049	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7050	{
7051	Assert(!fBlockSti);
7052	Assert(!fBlockMovSS);
7053	}
7054	#endif
7055	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
7056	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7057	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7058	AssertRCReturn(rc, rc);
7059
7060	/* Update the interruptibility-state as it could have been changed by
7061	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7062	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7063	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7064
7065	#ifdef VBOX_WITH_STATISTICS
7066	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7067	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7068	else
7069	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7070	#endif
7071	}
7072
7073	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7074	if ( !pVCpu->hm.s.fSingleInstruction
7075	&& !DBGFIsStepping(pVCpu))
7076	{
7077	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7078	AssertRCReturn(rc2, rc2);
7079	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7080	{
7081	/*
7082	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7083	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7084	* See Intel spec. 27.3.4 "Saving Non-Register State".
7085	*/
7086	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7087	AssertRCReturn(rc2, rc2);
7088	}
7089	}
7090	else
7091	{
7092	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
7093	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
7094	uIntrState = 0;
7095	}
7096
7097	/*
7098	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7099	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7100	*/
7101	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7102	AssertRC(rc2);
7103
7104	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7105	NOREF(fBlockMovSS); NOREF(fBlockSti);
7106	return rc;
7107	}
7108
7109
7110	/**
7111	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7112	*
7113	* @param pVCpu Pointer to the VMCPU.
7114	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7115	* out-of-sync. Make sure to update the required fields
7116	* before using them.
7117	*/
7118	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7119	{
7120	NOREF(pMixedCtx);
7121	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7122	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7123	}
7124
7125
7126	/**
7127	* Injects a double-fault (#DF) exception into the VM.
7128	*
7129	* @returns VBox status code (informational status code included).
7130	* @param pVCpu Pointer to the VMCPU.
7131	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7132	* out-of-sync. Make sure to update the required fields
7133	* before using them.
7134	*/
7135	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7136	{
7137	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7138	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7139	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7140	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7141	puIntrState);
7142	}
7143
7144
7145	/**
7146	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7147	*
7148	* @param pVCpu Pointer to the VMCPU.
7149	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7150	* out-of-sync. Make sure to update the required fields
7151	* before using them.
7152	*/
7153	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7154	{
7155	NOREF(pMixedCtx);
7156	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7157	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7158	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7159	}
7160
7161
7162	/**
7163	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7164	*
7165	* @param pVCpu Pointer to the VMCPU.
7166	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7167	* out-of-sync. Make sure to update the required fields
7168	* before using them.
7169	* @param cbInstr The value of RIP that is to be pushed on the guest
7170	* stack.
7171	*/
7172	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7173	{
7174	NOREF(pMixedCtx);
7175	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7176	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7177	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7178	}
7179
7180
7181	/**
7182	* Injects a general-protection (#GP) fault into the VM.
7183	*
7184	* @returns VBox status code (informational status code included).
7185	* @param pVCpu Pointer to the VMCPU.
7186	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7187	* out-of-sync. Make sure to update the required fields
7188	* before using them.
7189	* @param u32ErrorCode The error code associated with the #GP.
7190	*/
7191	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7192	uint32_t *puIntrState)
7193	{
7194	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7195	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7196	if (fErrorCodeValid)
7197	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7198	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7199	puIntrState);
7200	}
7201
7202
7203	/**
7204	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7205	*
7206	* @param pVCpu Pointer to the VMCPU.
7207	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7208	* out-of-sync. Make sure to update the required fields
7209	* before using them.
7210	* @param uVector The software interrupt vector number.
7211	* @param cbInstr The value of RIP that is to be pushed on the guest
7212	* stack.
7213	*/
7214	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7215	{
7216	NOREF(pMixedCtx);
7217	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7218	if ( uVector == X86_XCPT_BP
7219	\|\| uVector == X86_XCPT_OF)
7220	{
7221	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7222	}
7223	else
7224	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7225	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7226	}
7227
7228
7229	/**
7230	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7231	* stack.
7232	*
7233	* @returns VBox status code (information status code included).
7234	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7235	* @param pVM Pointer to the VM.
7236	* @param pMixedCtx Pointer to the guest-CPU context.
7237	* @param uValue The value to push to the guest stack.
7238	*/
7239	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7240	{
7241	/*
7242	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7243	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7244	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7245	*/
7246	if (pMixedCtx->sp == 1)
7247	return VINF_EM_RESET;
7248	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7249	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7250	AssertRCReturn(rc, rc);
7251	return rc;
7252	}
7253
7254
7255	/**
7256	* Injects an event into the guest upon VM-entry by updating the relevant fields
7257	* in the VM-entry area in the VMCS.
7258	*
7259	* @returns VBox status code (informational error codes included).
7260	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7261	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7262	*
7263	* @param pVCpu Pointer to the VMCPU.
7264	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7265	* be out-of-sync. Make sure to update the required
7266	* fields before using them.
7267	* @param u64IntInfo The VM-entry interruption-information field.
7268	* @param cbInstr The VM-entry instruction length in bytes (for
7269	* software interrupts, exceptions and privileged
7270	* software exceptions).
7271	* @param u32ErrCode The VM-entry exception error code.
7272	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7273	* @param puIntrState Pointer to the current guest interruptibility-state.
7274	* This interruptibility-state will be updated if
7275	* necessary. This cannot not be NULL.
7276	*
7277	* @remarks Requires CR0!
7278	* @remarks No-long-jump zone!!!
7279	*/
7280	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7281	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7282	{
7283	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7284	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7285	Assert(puIntrState);
7286	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7287
7288	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7289	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7290
7291	#ifdef VBOX_STRICT
7292	/* Validate the error-code-valid bit for hardware exceptions. */
7293	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7294	{
7295	switch (uVector)
7296	{
7297	case X86_XCPT_PF:
7298	case X86_XCPT_DF:
7299	case X86_XCPT_TS:
7300	case X86_XCPT_NP:
7301	case X86_XCPT_SS:
7302	case X86_XCPT_GP:
7303	case X86_XCPT_AC:
7304	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7305	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7306	/* fallthru */
7307	default:
7308	break;
7309	}
7310	}
7311	#endif
7312
7313	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7314	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7315	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7316
7317	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7318
7319	/* We require CR0 to check if the guest is in real-mode. */
7320	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7321	AssertRCReturn(rc, rc);
7322
7323	/*
7324	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7325	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7326	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7327	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7328	*/
7329	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7330	{
7331	PVM pVM = pVCpu->CTX_SUFF(pVM);
7332	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7333	{
7334	Assert(PDMVmmDevHeapIsEnabled(pVM));
7335	Assert(pVM->hm.s.vmx.pRealModeTSS);
7336
7337	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7338	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7339	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7340	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7341	AssertRCReturn(rc, rc);
7342	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
7343
7344	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7345	const size_t cbIdtEntry = sizeof(X86IDTR16);
7346	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7347	{
7348	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7349	if (uVector == X86_XCPT_DF)
7350	return VINF_EM_RESET;
7351	else if (uVector == X86_XCPT_GP)
7352	{
7353	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7354	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7355	}
7356
7357	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7358	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7359	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7360	}
7361
7362	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7363	uint16_t uGuestIp = pMixedCtx->ip;
7364	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7365	{
7366	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7367	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7368	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7369	}
7370	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7371	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7372
7373	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7374	X86IDTR16 IdtEntry;
7375	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7376	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7377	AssertRCReturn(rc, rc);
7378
7379	/* Construct the stack frame for the interrupt/exception handler. */
7380	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7381	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7382	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7383	AssertRCReturn(rc, rc);
7384
7385	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7386	if (rc == VINF_SUCCESS)
7387	{
7388	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7389	pMixedCtx->rip = IdtEntry.offSel;
7390	pMixedCtx->cs.Sel = IdtEntry.uSel;
7391	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7392	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7393	&& uVector == X86_XCPT_PF)
7394	{
7395	pMixedCtx->cr2 = GCPtrFaultAddress;
7396	}
7397
7398	/* If any other guest-state bits are changed here, make sure to update
7399	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7400	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7401	\| HM_CHANGED_GUEST_RIP
7402	\| HM_CHANGED_GUEST_RFLAGS
7403	\| HM_CHANGED_GUEST_RSP);
7404
7405	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7406	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7407	{
7408	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7409	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7410	Log4(("Clearing inhibition due to STI.\n"));
7411	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7412	}
7413	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7414
7415	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7416	it, if we are returning to ring-3 before executing guest code. */
7417	pVCpu->hm.s.Event.fPending = false;
7418	}
7419	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7420	return rc;
7421	}
7422	else
7423	{
7424	/*
7425	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7426	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7427	*/
7428	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7429	}
7430	}
7431
7432	/* Validate. */
7433	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7434	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7435	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7436
7437	/* Inject. */
7438	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7439	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7440	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7441	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7442
7443	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7444	&& uVector == X86_XCPT_PF)
7445	{
7446	pMixedCtx->cr2 = GCPtrFaultAddress;
7447	}
7448
7449	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7450	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7451
7452	AssertRCReturn(rc, rc);
7453	return rc;
7454	}
7455
7456
7457	/**
7458	* Clears the interrupt-window exiting control in the VMCS and if necessary
7459	* clears the current event in the VMCS as well.
7460	*
7461	* @returns VBox status code.
7462	* @param pVCpu Pointer to the VMCPU.
7463	*
7464	* @remarks Use this function only to clear events that have not yet been
7465	* delivered to the guest but are injected in the VMCS!
7466	* @remarks No-long-jump zone!!!
7467	*/
7468	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7469	{
7470	int rc;
7471	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7472
7473	/* Clear interrupt-window exiting control. */
7474	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7475	{
7476	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7477	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7478	AssertRC(rc);
7479	}
7480
7481	if (!pVCpu->hm.s.Event.fPending)
7482	return;
7483
7484	#ifdef VBOX_STRICT
7485	uint32_t u32EntryInfo;
7486	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7487	AssertRC(rc);
7488	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7489	#endif
7490
7491	/* Clear the entry-interruption field (including the valid bit). */
7492	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7493	AssertRC(rc);
7494
7495	/* Clear the pending debug exception field. */
7496	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7497	AssertRC(rc);
7498	}
7499
7500
7501	/**
7502	* Enters the VT-x session.
7503	*
7504	* @returns VBox status code.
7505	* @param pVM Pointer to the VM.
7506	* @param pVCpu Pointer to the VMCPU.
7507	* @param pCpu Pointer to the CPU info struct.
7508	*/
7509	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7510	{
7511	AssertPtr(pVM);
7512	AssertPtr(pVCpu);
7513	Assert(pVM->hm.s.vmx.fSupported);
7514	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7515	NOREF(pCpu); NOREF(pVM);
7516
7517	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7518	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7519
7520	#ifdef VBOX_STRICT
7521	/* Make sure we're in VMX root mode. */
7522	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7523	if (!(u32HostCR4 & X86_CR4_VMXE))
7524	{
7525	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7526	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7527	}
7528	#endif
7529
7530	/*
7531	* Load the VCPU's VMCS as the current (and active) one.
7532	*/
7533	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7534	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7535	if (RT_FAILURE(rc))
7536	return rc;
7537
7538	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7539	pVCpu->hm.s.fLeaveDone = false;
7540	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7541
7542	return VINF_SUCCESS;
7543	}
7544
7545
7546	/**
7547	* The thread-context callback (only on platforms which support it).
7548	*
7549	* @param enmEvent The thread-context event.
7550	* @param pVCpu Pointer to the VMCPU.
7551	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7552	* @thread EMT(pVCpu)
7553	*/
7554	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7555	{
7556	NOREF(fGlobalInit);
7557
7558	switch (enmEvent)
7559	{
7560	case RTTHREADCTXEVENT_PREEMPTING:
7561	{
7562	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7563	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7564	VMCPU_ASSERT_EMT(pVCpu);
7565
7566	PVM pVM = pVCpu->CTX_SUFF(pVM);
7567	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7568
7569	/* No longjmps (logger flushes, locks) in this fragile context. */
7570	VMMRZCallRing3Disable(pVCpu);
7571	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7572
7573	/*
7574	* Restore host-state (FPU, debug etc.)
7575	*/
7576	if (!pVCpu->hm.s.fLeaveDone)
7577	{
7578	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7579	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7580	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7581	pVCpu->hm.s.fLeaveDone = true;
7582	}
7583
7584	/* Leave HM context, takes care of local init (term). */
7585	int rc = HMR0LeaveCpu(pVCpu);
7586	AssertRC(rc); NOREF(rc);
7587
7588	/* Restore longjmp state. */
7589	VMMRZCallRing3Enable(pVCpu);
7590	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7591	break;
7592	}
7593
7594	case RTTHREADCTXEVENT_RESUMED:
7595	{
7596	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7597	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7598	VMCPU_ASSERT_EMT(pVCpu);
7599
7600	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7601	VMMRZCallRing3Disable(pVCpu);
7602	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7603
7604	/* Initialize the bare minimum state required for HM. This takes care of
7605	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7606	int rc = HMR0EnterCpu(pVCpu);
7607	AssertRC(rc);
7608	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7609
7610	/* Load the active VMCS as the current one. */
7611	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7612	{
7613	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7614	AssertRC(rc); NOREF(rc);
7615	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7616	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7617	}
7618	pVCpu->hm.s.fLeaveDone = false;
7619
7620	/* Restore longjmp state. */
7621	VMMRZCallRing3Enable(pVCpu);
7622	break;
7623	}
7624
7625	default:
7626	break;
7627	}
7628	}
7629
7630
7631	/**
7632	* Saves the host state in the VMCS host-state.
7633	* Sets up the VM-exit MSR-load area.
7634	*
7635	* The CPU state will be loaded from these fields on every successful VM-exit.
7636	*
7637	* @returns VBox status code.
7638	* @param pVM Pointer to the VM.
7639	* @param pVCpu Pointer to the VMCPU.
7640	*
7641	* @remarks No-long-jump zone!!!
7642	*/
7643	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7644	{
7645	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7646
7647	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7648	return VINF_SUCCESS;
7649
7650	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7651	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7652
7653	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7654	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7655
7656	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7657	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7658
7659	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7660	return rc;
7661	}
7662
7663
7664	/**
7665	* Saves the host state in the VMCS host-state.
7666	*
7667	* @returns VBox status code.
7668	* @param pVM Pointer to the VM.
7669	* @param pVCpu Pointer to the VMCPU.
7670	*
7671	* @remarks No-long-jump zone!!!
7672	*/
7673	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7674	{
7675	AssertPtr(pVM);
7676	AssertPtr(pVCpu);
7677
7678	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7679
7680	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7681	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7682	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7683	return hmR0VmxSaveHostState(pVM, pVCpu);
7684	}
7685
7686
7687	/**
7688	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7689	* loaded from these fields on every successful VM-entry.
7690	*
7691	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7692	* Sets up the VM-entry controls.
7693	* Sets up the appropriate VMX non-root function to execute guest code based on
7694	* the guest CPU mode.
7695	*
7696	* @returns VBox status code.
7697	* @param pVM Pointer to the VM.
7698	* @param pVCpu Pointer to the VMCPU.
7699	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7700	* out-of-sync. Make sure to update the required fields
7701	* before using them.
7702	*
7703	* @remarks No-long-jump zone!!!
7704	*/
7705	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7706	{
7707	AssertPtr(pVM);
7708	AssertPtr(pVCpu);
7709	AssertPtr(pMixedCtx);
7710	HMVMX_ASSERT_PREEMPT_SAFE();
7711
7712	#ifdef LOG_ENABLED
7713	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7714	* probably not initialized yet? Anyway this will do for now.
7715	*
7716	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7717	* interface and disable ring-3 calls when thread-context hooks are not
7718	* available. */
7719	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7720	VMMR0LogFlushDisable(pVCpu);
7721	#endif
7722
7723	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7724
7725	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7726
7727	/* Determine real-on-v86 mode. */
7728	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7729	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7730	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7731	{
7732	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7733	}
7734
7735	/*
7736	* Load the guest-state into the VMCS.
7737	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7738	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7739	*/
7740	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7741	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7742
7743	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7744	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7745	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7746
7747	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7748	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7749	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7750
7751	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7752	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7753
7754	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7755	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7756
7757	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7758	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7759	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7760
7761	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7762	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7763
7764	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7765	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7766
7767	/*
7768	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7769	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7770	*/
7771	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7772	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7773
7774	/* Clear any unused and reserved bits. */
7775	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7776
7777	#ifdef LOG_ENABLED
7778	/* Only reenable log-flushing if the caller has it enabled. */
7779	if (!fCallerDisabledLogFlush)
7780	VMMR0LogFlushEnable(pVCpu);
7781	#endif
7782
7783	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7784	return rc;
7785	}
7786
7787
7788	/**
7789	* Loads the state shared between the host and guest into the VMCS.
7790	*
7791	* @param pVM Pointer to the VM.
7792	* @param pVCpu Pointer to the VMCPU.
7793	* @param pCtx Pointer to the guest-CPU context.
7794	*
7795	* @remarks No-long-jump zone!!!
7796	*/
7797	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7798	{
7799	NOREF(pVM);
7800
7801	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7802	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7803
7804	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7805	{
7806	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7807	AssertRC(rc);
7808	}
7809
7810	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7811	{
7812	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7813	AssertRC(rc);
7814
7815	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7816	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7817	{
7818	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7819	AssertRC(rc);
7820	}
7821	}
7822
7823	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7824	{
7825	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7826	if ( HMVMX_IS_64BIT_HOST_MODE()
7827	&& pVM->hm.s.fAllow64BitGuests)
7828	{
7829	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7830	}
7831	#endif
7832	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7833	}
7834
7835	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7836	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7837	}
7838
7839
7840	/**
7841	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7842	*
7843	* @param pVM Pointer to the VM.
7844	* @param pVCpu Pointer to the VMCPU.
7845	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7846	* out-of-sync. Make sure to update the required fields
7847	* before using them.
7848	*/
7849	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7850	{
7851	HMVMX_ASSERT_PREEMPT_SAFE();
7852
7853	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7854	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7855	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7856	#endif
7857
7858	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7859	{
7860	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7861	AssertRC(rc);
7862	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7863	}
7864	else if (HMCPU_CF_VALUE(pVCpu))
7865	{
7866	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7867	AssertRC(rc);
7868	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7869	}
7870
7871	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7872	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7873	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7874	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7875
7876	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7877	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7878	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7879	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7880	#endif
7881	}
7882
7883
7884	/**
7885	* Does the preparations before executing guest code in VT-x.
7886	*
7887	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7888	* recompiler. We must be cautious what we do here regarding committing
7889	* guest-state information into the VMCS assuming we assuredly execute the
7890	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7891	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7892	* so that the recompiler can (and should) use them when it resumes guest
7893	* execution. Otherwise such operations must be done when we can no longer
7894	* exit to ring-3.
7895	*
7896	* @returns Strict VBox status code.
7897	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7898	* have been disabled.
7899	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7900	* double-fault into the guest.
7901	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7902	*
7903	* @param pVM Pointer to the VM.
7904	* @param pVCpu Pointer to the VMCPU.
7905	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7906	* out-of-sync. Make sure to update the required fields
7907	* before using them.
7908	* @param pVmxTransient Pointer to the VMX transient structure.
7909	*/
7910	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7911	{
7912	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7913
7914	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7915	PGMRZDynMapFlushAutoSet(pVCpu);
7916	#endif
7917
7918	/* Check force flag actions that might require us to go back to ring-3. */
7919	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7920	if (rc != VINF_SUCCESS)
7921	return rc;
7922
7923	#ifndef IEM_VERIFICATION_MODE_FULL
7924	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7925	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7926	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7927	{
7928	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7929	RTGCPHYS GCPhysApicBase;
7930	GCPhysApicBase = pMixedCtx->msrApicBase;
7931	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7932
7933	/* Unalias any existing mapping. */
7934	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7935	AssertRCReturn(rc, rc);
7936
7937	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7938	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7939	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7940	AssertRCReturn(rc, rc);
7941
7942	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7943	}
7944	#endif /* !IEM_VERIFICATION_MODE_FULL */
7945
7946	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7947	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7948
7949	/*
7950	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7951	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7952	*/
7953	if (TRPMHasTrap(pVCpu))
7954	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7955	else if (!pVCpu->hm.s.Event.fPending)
7956	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7957
7958	/*
7959	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7960	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7961	*/
7962	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7963	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7964	{
7965	Assert(rc == VINF_EM_RESET);
7966	return rc;
7967	}
7968
7969	/*
7970	* No longjmps to ring-3 from this point on!!!
7971	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7972	* This also disables flushing of the R0-logger instance (if any).
7973	*/
7974	VMMRZCallRing3Disable(pVCpu);
7975
7976	/*
7977	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7978	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7979	*
7980	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7981	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7982	*
7983	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7984	* executing guest code.
7985	*/
7986	pVmxTransient->uEflags = ASMIntDisableFlags();
7987	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7988	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7989	{
7990	hmR0VmxClearEventVmcs(pVCpu);
7991	ASMSetFlags(pVmxTransient->uEflags);
7992	VMMRZCallRing3Enable(pVCpu);
7993	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7994	return VINF_EM_RAW_TO_R3;
7995	}
7996	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7997	{
7998	hmR0VmxClearEventVmcs(pVCpu);
7999	ASMSetFlags(pVmxTransient->uEflags);
8000	VMMRZCallRing3Enable(pVCpu);
8001	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8002	return VINF_EM_RAW_INTERRUPT;
8003	}
8004
8005	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8006	pVCpu->hm.s.Event.fPending = false;
8007
8008	return VINF_SUCCESS;
8009	}
8010
8011
8012	/**
8013	* Prepares to run guest code in VT-x and we've committed to doing so. This
8014	* means there is no backing out to ring-3 or anywhere else at this
8015	* point.
8016	*
8017	* @param pVM Pointer to the VM.
8018	* @param pVCpu Pointer to the VMCPU.
8019	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8020	* out-of-sync. Make sure to update the required fields
8021	* before using them.
8022	* @param pVmxTransient Pointer to the VMX transient structure.
8023	*
8024	* @remarks Called with preemption disabled.
8025	* @remarks No-long-jump zone!!!
8026	*/
8027	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8028	{
8029	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8030	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8031	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8032
8033	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8034	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8035
8036	/*
8037	* If we are injecting events to a real-on-v86 mode guest, we may have to update
8038	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8039	* Reload only the necessary state, the assertion will catch if other parts of the code
8040	* change.
8041	*/
8042	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8043	{
8044	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8045	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8046	}
8047
8048	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8049	if (!CPUMIsGuestFPUStateActive(pVCpu))
8050	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8051	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8052	#endif
8053
8054	if ( pVCpu->hm.s.fUseGuestFpu
8055	&& !CPUMIsGuestFPUStateActive(pVCpu))
8056	{
8057	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8058	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
8059	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8060	}
8061
8062	/*
8063	* The host MSR values the very first time around won't be updated, so we need to
8064	* fill those values in. Subsequently, it's updated as part of the host state.
8065	*/
8066	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8067	&& pVCpu->hm.s.vmx.cMsrs > 0)
8068	{
8069	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8070	}
8071
8072	/*
8073	* Load the host state bits as we may've been preempted (only happens when
8074	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8075	*/
8076	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8077	{
8078	/* This ASSUMES that pfnStartVM has been set up already. */
8079	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8080	AssertRC(rc);
8081	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8082	}
8083	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8084
8085	/*
8086	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8087	*/
8088	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8089	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8090	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8091
8092	/* Store status of the shared guest-host state at the time of VM-entry. */
8093	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8094	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8095	{
8096	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8097	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8098	}
8099	else
8100	#endif
8101	{
8102	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8103	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8104	}
8105	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8106
8107	/*
8108	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8109	*/
8110	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8111	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8112
8113	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8114	RTCPUID idCurrentCpu = pCpu->idCpu;
8115	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8116	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8117	{
8118	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8119	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8120	}
8121
8122	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8123	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8124	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8125	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8126
8127	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8128
8129	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8130	to start executing. */
8131
8132	/*
8133	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8134	*/
8135	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8136	{
8137	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8138	{
8139	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8140	AssertRC(rc2);
8141	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
8142	uint64_t u64GuestTscAuxMsr;
8143	rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTscAuxMsr);
8144	AssertRC(rc2);
8145	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, u64GuestTscAuxMsr, true /* fUpdateHostMsr */);
8146	}
8147	else
8148	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8149	}
8150	#ifdef VBOX_STRICT
8151	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8152	#endif
8153	}
8154
8155
8156	/**
8157	* Performs some essential restoration of state after running guest code in
8158	* VT-x.
8159	*
8160	* @param pVM Pointer to the VM.
8161	* @param pVCpu Pointer to the VMCPU.
8162	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8163	* out-of-sync. Make sure to update the required fields
8164	* before using them.
8165	* @param pVmxTransient Pointer to the VMX transient structure.
8166	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8167	*
8168	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8169	*
8170	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8171	* unconditionally when it is safe to do so.
8172	*/
8173	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8174	{
8175	NOREF(pVM);
8176
8177	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8178
8179	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8180	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8181	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8182	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8183	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8184
8185	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8186	{
8187	/** @todo Find a way to fix hardcoding a guestimate. */
8188	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8189	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8190	}
8191
8192	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8193	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8194	Assert(!(ASMGetFlags() & X86_EFL_IF));
8195	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8196
8197	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8198	if (CPUMIsGuestFPUStateActive(pVCpu))
8199	{
8200	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8201	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8202	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8203	}
8204	#endif
8205
8206	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8207	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8208	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8209	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8210
8211	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8212	uint32_t uExitReason;
8213	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8214	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8215	AssertRC(rc);
8216	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8217	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8218
8219	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8220	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8221	{
8222	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8223	pVmxTransient->fVMEntryFailed));
8224	return;
8225	}
8226
8227	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8228	{
8229	/* Update the guest interruptibility-state from the VMCS. */
8230	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8231	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8232	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8233	AssertRC(rc);
8234	#endif
8235	/*
8236	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8237	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8238	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8239	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8240	*/
8241	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8242	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8243	{
8244	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8245	AssertRC(rc);
8246	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8247	}
8248	}
8249	}
8250
8251
8252
8253	/**
8254	* Runs the guest code using VT-x the normal way.
8255	*
8256	* @returns VBox status code.
8257	* @param pVM Pointer to the VM.
8258	* @param pVCpu Pointer to the VMCPU.
8259	* @param pCtx Pointer to the guest-CPU context.
8260	*
8261	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8262	*/
8263	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8264	{
8265	VMXTRANSIENT VmxTransient;
8266	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8267	int rc = VERR_INTERNAL_ERROR_5;
8268	uint32_t cLoops = 0;
8269
8270	for (;; cLoops++)
8271	{
8272	Assert(!HMR0SuspendPending());
8273	HMVMX_ASSERT_CPU_SAFE();
8274
8275	/* Preparatory work for running guest code, this may force us to return
8276	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8277	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8278	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8279	if (rc != VINF_SUCCESS)
8280	break;
8281
8282	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8283	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8284	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8285
8286	/* Restore any residual host-state and save any bits shared between host
8287	and guest into the guest-CPU state. Re-enables interrupts! */
8288	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8289
8290	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8291	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8292	{
8293	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8294	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8295	return rc;
8296	}
8297
8298	/* Handle the VM-exit. */
8299	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8300	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8301	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8302	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8303	HMVMX_START_EXIT_DISPATCH_PROF();
8304	#ifdef HMVMX_USE_FUNCTION_TABLE
8305	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8306	#else
8307	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8308	#endif
8309	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8310	if (rc != VINF_SUCCESS)
8311	break;
8312	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8313	{
8314	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8315	rc = VINF_EM_RAW_INTERRUPT;
8316	break;
8317	}
8318	}
8319
8320	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8321	return rc;
8322	}
8323
8324
8325	/**
8326	* Single steps guest code using VT-x.
8327	*
8328	* @returns VBox status code.
8329	* @param pVM Pointer to the VM.
8330	* @param pVCpu Pointer to the VMCPU.
8331	* @param pCtx Pointer to the guest-CPU context.
8332	*
8333	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8334	*/
8335	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8336	{
8337	VMXTRANSIENT VmxTransient;
8338	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8339	int rc = VERR_INTERNAL_ERROR_5;
8340	uint32_t cLoops = 0;
8341	uint16_t uCsStart = pCtx->cs.Sel;
8342	uint64_t uRipStart = pCtx->rip;
8343
8344	for (;; cLoops++)
8345	{
8346	Assert(!HMR0SuspendPending());
8347	HMVMX_ASSERT_CPU_SAFE();
8348
8349	/* Preparatory work for running guest code, this may force us to return
8350	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8351	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8352	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8353	if (rc != VINF_SUCCESS)
8354	break;
8355
8356	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8357	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8358	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8359
8360	/* Restore any residual host-state and save any bits shared between host
8361	and guest into the guest-CPU state. Re-enables interrupts! */
8362	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8363
8364	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8365	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8366	{
8367	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8368	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8369	return rc;
8370	}
8371
8372	/* Handle the VM-exit. */
8373	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8374	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8375	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8376	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8377	HMVMX_START_EXIT_DISPATCH_PROF();
8378	#ifdef HMVMX_USE_FUNCTION_TABLE
8379	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8380	#else
8381	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8382	#endif
8383	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8384	if (rc != VINF_SUCCESS)
8385	break;
8386	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8387	{
8388	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8389	rc = VINF_EM_RAW_INTERRUPT;
8390	break;
8391	}
8392
8393	/*
8394	* Did the RIP change, if so, consider it a single step.
8395	* Otherwise, make sure one of the TFs gets set.
8396	*/
8397	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8398	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8399	AssertRCReturn(rc2, rc2);
8400	if ( pCtx->rip != uRipStart
8401	\|\| pCtx->cs.Sel != uCsStart)
8402	{
8403	rc = VINF_EM_DBG_STEPPED;
8404	break;
8405	}
8406	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8407	}
8408
8409	/*
8410	* Clear the X86_EFL_TF if necessary.
8411	*/
8412	if (pVCpu->hm.s.fClearTrapFlag)
8413	{
8414	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8415	AssertRCReturn(rc2, rc2);
8416	pVCpu->hm.s.fClearTrapFlag = false;
8417	pCtx->eflags.Bits.u1TF = 0;
8418	}
8419	/** @todo there seems to be issues with the resume flag when the monitor trap
8420	* flag is pending without being used. Seen early in bios init when
8421	* accessing APIC page in prot mode. */
8422
8423	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8424	return rc;
8425	}
8426
8427
8428	/**
8429	* Runs the guest code using VT-x.
8430	*
8431	* @returns VBox status code.
8432	* @param pVM Pointer to the VM.
8433	* @param pVCpu Pointer to the VMCPU.
8434	* @param pCtx Pointer to the guest-CPU context.
8435	*/
8436	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8437	{
8438	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8439	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8440	HMVMX_ASSERT_PREEMPT_SAFE();
8441
8442	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8443
8444	int rc;
8445	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8446	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8447	else
8448	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8449
8450	if (rc == VERR_EM_INTERPRETER)
8451	rc = VINF_EM_RAW_EMULATE_INSTR;
8452	else if (rc == VINF_EM_RESET)
8453	rc = VINF_EM_TRIPLE_FAULT;
8454
8455	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8456	if (RT_FAILURE(rc2))
8457	{
8458	pVCpu->hm.s.u32HMError = rc;
8459	rc = rc2;
8460	}
8461	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8462	return rc;
8463	}
8464
8465
8466	#ifndef HMVMX_USE_FUNCTION_TABLE
8467	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8468	{
8469	#ifdef DEBUG_ramshankar
8470	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8471	# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8472	#endif
8473	int rc;
8474	switch (rcReason)
8475	{
8476	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8477	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8478	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8479	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8480	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8481	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8482	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8483	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8484	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8485	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8486	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8487	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8488	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8489	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8490	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8491	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8492	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8493	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8494	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8495	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8496	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8497	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8498	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8499	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8500	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8501	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8502	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8503	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8504	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8505	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8506	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8507	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8508	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8509
8510	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8511	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8512	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8513	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8514	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8515	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8516	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8517	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8518	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8519
8520	case VMX_EXIT_VMCALL:
8521	case VMX_EXIT_VMCLEAR:
8522	case VMX_EXIT_VMLAUNCH:
8523	case VMX_EXIT_VMPTRLD:
8524	case VMX_EXIT_VMPTRST:
8525	case VMX_EXIT_VMREAD:
8526	case VMX_EXIT_VMRESUME:
8527	case VMX_EXIT_VMWRITE:
8528	case VMX_EXIT_VMXOFF:
8529	case VMX_EXIT_VMXON:
8530	case VMX_EXIT_INVEPT:
8531	case VMX_EXIT_INVVPID:
8532	case VMX_EXIT_VMFUNC:
8533	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8534	break;
8535	default:
8536	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8537	break;
8538	}
8539	return rc;
8540	}
8541	#endif
8542
8543	#ifdef DEBUG
8544	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8545	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8546	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8547
8548	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8549	do \
8550	{ \
8551	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8552	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8553	} while (0)
8554
8555	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8556	do { \
8557	AssertPtr(pVCpu); \
8558	AssertPtr(pMixedCtx); \
8559	AssertPtr(pVmxTransient); \
8560	Assert(pVmxTransient->fVMEntryFailed == false); \
8561	Assert(ASMIntAreEnabled()); \
8562	HMVMX_ASSERT_PREEMPT_SAFE(); \
8563	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8564	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8565	HMVMX_ASSERT_PREEMPT_SAFE(); \
8566	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8567	HMVMX_ASSERT_PREEMPT_CPUID(); \
8568	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8569	} while (0)
8570
8571	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8572	do { \
8573	Log4Func(("\n")); \
8574	} while (0)
8575	#else /* Release builds */
8576	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8577	do { \
8578	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8579	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8580	} while (0)
8581	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8582	#endif
8583
8584
8585	/**
8586	* Advances the guest RIP after reading it from the VMCS.
8587	*
8588	* @returns VBox status code.
8589	* @param pVCpu Pointer to the VMCPU.
8590	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8591	* out-of-sync. Make sure to update the required fields
8592	* before using them.
8593	* @param pVmxTransient Pointer to the VMX transient structure.
8594	*
8595	* @remarks No-long-jump zone!!!
8596	*/
8597	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8598	{
8599	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8600	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8601	AssertRCReturn(rc, rc);
8602
8603	pMixedCtx->rip += pVmxTransient->cbInstr;
8604	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8605	return rc;
8606	}
8607
8608
8609	/**
8610	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8611	* and update error record fields accordingly.
8612	*
8613	* @return VMX_IGS_* return codes.
8614	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8615	* wrong with the guest state.
8616	*
8617	* @param pVM Pointer to the VM.
8618	* @param pVCpu Pointer to the VMCPU.
8619	* @param pCtx Pointer to the guest-CPU state.
8620	*/
8621	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8622	{
8623	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8624	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8625	uError = (err); \
8626	break; \
8627	} else do { } while (0)
8628
8629	int rc;
8630	uint32_t uError = VMX_IGS_ERROR;
8631	uint32_t u32Val;
8632	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8633
8634	do
8635	{
8636	/*
8637	* CR0.
8638	*/
8639	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8640	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8641	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8642	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8643	if (fUnrestrictedGuest)
8644	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8645
8646	uint32_t u32GuestCR0;
8647	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8648	AssertRCBreak(rc);
8649	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8650	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8651	if ( !fUnrestrictedGuest
8652	&& (u32GuestCR0 & X86_CR0_PG)
8653	&& !(u32GuestCR0 & X86_CR0_PE))
8654	{
8655	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8656	}
8657
8658	/*
8659	* CR4.
8660	*/
8661	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8662	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8663
8664	uint32_t u32GuestCR4;
8665	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8666	AssertRCBreak(rc);
8667	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8668	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8669
8670	/*
8671	* IA32_DEBUGCTL MSR.
8672	*/
8673	uint64_t u64Val;
8674	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8675	AssertRCBreak(rc);
8676	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8677	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8678	{
8679	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8680	}
8681	uint64_t u64DebugCtlMsr = u64Val;
8682
8683	#ifdef VBOX_STRICT
8684	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8685	AssertRCBreak(rc);
8686	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8687	#endif
8688	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8689
8690	/*
8691	* RIP and RFLAGS.
8692	*/
8693	uint32_t u32Eflags;
8694	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8695	if (HMVMX_IS_64BIT_HOST_MODE())
8696	{
8697	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8698	AssertRCBreak(rc);
8699	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8700	if ( !fLongModeGuest
8701	\|\| !pCtx->cs.Attr.n.u1Long)
8702	{
8703	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8704	}
8705	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8706	* must be identical if the "IA32e mode guest" VM-entry control is 1
8707	* and CS.L is 1. No check applies if the CPU supports 64
8708	* linear-address bits. */
8709
8710	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8711	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8712	AssertRCBreak(rc);
8713	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8714	VMX_IGS_RFLAGS_RESERVED);
8715	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8716	u32Eflags = u64Val;
8717	}
8718	else
8719	#endif
8720	{
8721	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8722	AssertRCBreak(rc);
8723	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8724	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8725	}
8726
8727	if ( fLongModeGuest
8728	\|\| ( fUnrestrictedGuest
8729	&& !(u32GuestCR0 & X86_CR0_PE)))
8730	{
8731	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8732	}
8733
8734	uint32_t u32EntryInfo;
8735	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8736	AssertRCBreak(rc);
8737	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8738	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8739	{
8740	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8741	}
8742
8743	/*
8744	* 64-bit checks.
8745	*/
8746	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8747	if (HMVMX_IS_64BIT_HOST_MODE())
8748	{
8749	if ( fLongModeGuest
8750	&& !fUnrestrictedGuest)
8751	{
8752	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8753	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8754	}
8755
8756	if ( !fLongModeGuest
8757	&& (u32GuestCR4 & X86_CR4_PCIDE))
8758	{
8759	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8760	}
8761
8762	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8763	* 51:32 beyond the processor's physical-address width are 0. */
8764
8765	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8766	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8767	{
8768	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8769	}
8770
8771	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8772	AssertRCBreak(rc);
8773	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8774
8775	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8776	AssertRCBreak(rc);
8777	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8778	}
8779	#endif
8780
8781	/*
8782	* PERF_GLOBAL MSR.
8783	*/
8784	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8785	{
8786	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8787	AssertRCBreak(rc);
8788	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8789	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8790	}
8791
8792	/*
8793	* PAT MSR.
8794	*/
8795	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8796	{
8797	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8798	AssertRCBreak(rc);
8799	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8800	for (unsigned i = 0; i < 8; i++)
8801	{
8802	uint8_t u8Val = (u64Val & 0x7);
8803	if ( u8Val != 0 /* UC */
8804	\|\| u8Val != 1 /* WC */
8805	\|\| u8Val != 4 /* WT */
8806	\|\| u8Val != 5 /* WP */
8807	\|\| u8Val != 6 /* WB */
8808	\|\| u8Val != 7 /* UC- */)
8809	{
8810	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8811	}
8812	u64Val >>= 3;
8813	}
8814	}
8815
8816	/*
8817	* EFER MSR.
8818	*/
8819	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8820	{
8821	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8822	AssertRCBreak(rc);
8823	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8824	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8825	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8826	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8827	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8828	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8829	}
8830
8831	/*
8832	* Segment registers.
8833	*/
8834	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8835	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8836	if (!(u32Eflags & X86_EFL_VM))
8837	{
8838	/* CS */
8839	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8840	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8841	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8842	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8843	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8844	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8845	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8846	/* CS cannot be loaded with NULL in protected mode. */
8847	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8848	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8849	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8850	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8851	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8852	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8853	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8854	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8855	else
8856	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8857
8858	/* SS */
8859	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8860	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8861	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8862	if ( !(pCtx->cr0 & X86_CR0_PE)
8863	\|\| pCtx->cs.Attr.n.u4Type == 3)
8864	{
8865	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8866	}
8867	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8868	{
8869	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8870	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8871	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8872	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8873	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8874	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8875	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8876	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8877	}
8878
8879	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8880	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8881	{
8882	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8883	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8884	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8885	\|\| pCtx->ds.Attr.n.u4Type > 11
8886	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8887	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8888	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8889	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8890	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8891	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8892	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8893	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8894	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8895	}
8896	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8897	{
8898	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8899	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8900	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8901	\|\| pCtx->es.Attr.n.u4Type > 11
8902	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8903	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8904	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8905	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8906	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8907	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8908	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8909	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8910	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8911	}
8912	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8913	{
8914	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8915	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8916	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8917	\|\| pCtx->fs.Attr.n.u4Type > 11
8918	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8919	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8920	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8921	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8922	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8923	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8924	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8925	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8926	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8927	}
8928	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8929	{
8930	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8931	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8932	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8933	\|\| pCtx->gs.Attr.n.u4Type > 11
8934	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8935	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8936	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8937	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8938	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8939	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8940	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8941	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8942	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8943	}
8944	/* 64-bit capable CPUs. */
8945	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8946	if (HMVMX_IS_64BIT_HOST_MODE())
8947	{
8948	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8949	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8950	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8951	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8952	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8953	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8954	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8955	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8956	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8957	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8958	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8959	}
8960	#endif
8961	}
8962	else
8963	{
8964	/* V86 mode checks. */
8965	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8966	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8967	{
8968	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8969	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8970	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8971	}
8972	else
8973	{
8974	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8975	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8976	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8977	}
8978
8979	/* CS */
8980	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8981	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8982	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8983	/* SS */
8984	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8985	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8986	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8987	/* DS */
8988	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8989	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8990	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8991	/* ES */
8992	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8993	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8994	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8995	/* FS */
8996	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8997	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8998	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8999	/* GS */
9000	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9001	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9002	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9003	/* 64-bit capable CPUs. */
9004	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9005	if (HMVMX_IS_64BIT_HOST_MODE())
9006	{
9007	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9008	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9009	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9010	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9011	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9012	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9013	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9014	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9015	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9016	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9017	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9018	}
9019	#endif
9020	}
9021
9022	/*
9023	* TR.
9024	*/
9025	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9026	/* 64-bit capable CPUs. */
9027	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9028	if (HMVMX_IS_64BIT_HOST_MODE())
9029	{
9030	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9031	}
9032	#endif
9033	if (fLongModeGuest)
9034	{
9035	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9036	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9037	}
9038	else
9039	{
9040	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9041	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9042	VMX_IGS_TR_ATTR_TYPE_INVALID);
9043	}
9044	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9045	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9046	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9047	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9048	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9049	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9050	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9051	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9052
9053	/*
9054	* GDTR and IDTR.
9055	*/
9056	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9057	if (HMVMX_IS_64BIT_HOST_MODE())
9058	{
9059	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9060	AssertRCBreak(rc);
9061	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9062
9063	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9064	AssertRCBreak(rc);
9065	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9066	}
9067	#endif
9068
9069	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9070	AssertRCBreak(rc);
9071	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9072
9073	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9074	AssertRCBreak(rc);
9075	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9076
9077	/*
9078	* Guest Non-Register State.
9079	*/
9080	/* Activity State. */
9081	uint32_t u32ActivityState;
9082	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9083	AssertRCBreak(rc);
9084	HMVMX_CHECK_BREAK( !u32ActivityState
9085	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9086	VMX_IGS_ACTIVITY_STATE_INVALID);
9087	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9088	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9089	uint32_t u32IntrState;
9090	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9091	AssertRCBreak(rc);
9092	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9093	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9094	{
9095	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9096	}
9097
9098	/** @todo Activity state and injecting interrupts. Left as a todo since we
9099	* currently don't use activity states but ACTIVE. */
9100
9101	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9102	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9103
9104	/* Guest interruptibility-state. */
9105	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9106	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9107	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9108	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9109	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9110	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9111	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9112	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9113	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9114	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9115	{
9116	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9117	{
9118	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9119	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9120	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9121	}
9122	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9123	{
9124	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9125	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9126	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9127	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9128	}
9129	}
9130	/** @todo Assumes the processor is not in SMM. */
9131	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9132	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9133	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9134	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9135	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9136	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9137	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9138	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9139	{
9140	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9141	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9142	}
9143
9144	/* Pending debug exceptions. */
9145	if (HMVMX_IS_64BIT_HOST_MODE())
9146	{
9147	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9148	AssertRCBreak(rc);
9149	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9150	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9151	u32Val = u64Val; /* For pending debug exceptions checks below. */
9152	}
9153	else
9154	{
9155	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9156	AssertRCBreak(rc);
9157	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9158	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9159	}
9160
9161	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9162	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9163	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9164	{
9165	if ( (u32Eflags & X86_EFL_TF)
9166	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9167	{
9168	/* Bit 14 is PendingDebug.BS. */
9169	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9170	}
9171	if ( !(u32Eflags & X86_EFL_TF)
9172	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9173	{
9174	/* Bit 14 is PendingDebug.BS. */
9175	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9176	}
9177	}
9178
9179	/* VMCS link pointer. */
9180	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9181	AssertRCBreak(rc);
9182	if (u64Val != UINT64_C(0xffffffffffffffff))
9183	{
9184	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9185	/** @todo Bits beyond the processor's physical-address width MBZ. */
9186	/** @todo 32-bit located in memory referenced by value of this field (as a
9187	* physical address) must contain the processor's VMCS revision ID. */
9188	/** @todo SMM checks. */
9189	}
9190
9191	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
9192
9193	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9194	if (uError == VMX_IGS_ERROR)
9195	uError = VMX_IGS_REASON_NOT_FOUND;
9196	} while (0);
9197
9198	pVCpu->hm.s.u32HMError = uError;
9199	return uError;
9200
9201	#undef HMVMX_ERROR_BREAK
9202	#undef HMVMX_CHECK_BREAK
9203	}
9204
9205	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9206	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9207	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9208
9209	/** @name VM-exit handlers.
9210	* @{
9211	*/
9212
9213	/**
9214	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9215	*/
9216	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9217	{
9218	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9219	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9220	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9221	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9222	return VINF_SUCCESS;
9223	return VINF_EM_RAW_INTERRUPT;
9224	}
9225
9226
9227	/**
9228	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9229	*/
9230	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9231	{
9232	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9233	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9234
9235	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9236	AssertRCReturn(rc, rc);
9237
9238	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9239	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9240	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9241	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9242
9243	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9244	{
9245	/*
9246	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9247	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9248	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9249	*
9250	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9251	*/
9252	VMXDispatchHostNmi();
9253	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9254	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9255	return VINF_SUCCESS;
9256	}
9257
9258	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9259	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9260	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9261	{
9262	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9263	return VINF_SUCCESS;
9264	}
9265	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9266	{
9267	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9268	return rc;
9269	}
9270
9271	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9272	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9273	switch (uIntType)
9274	{
9275	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9276	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9277	/* no break */
9278	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9279	{
9280	switch (uVector)
9281	{
9282	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9283	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9284	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9285	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9286	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9287	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9288	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9289	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9290	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9291	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9292	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9293	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9294	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9295	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9296	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9297	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9298	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9299	#endif
9300	default:
9301	{
9302	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9303	AssertRCReturn(rc, rc);
9304
9305	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9306	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9307	{
9308	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9309	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9310	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9311
9312	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9313	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9314	AssertRCReturn(rc, rc);
9315	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9316	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9317	0 /* GCPtrFaultAddress */);
9318	AssertRCReturn(rc, rc);
9319	}
9320	else
9321	{
9322	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9323	pVCpu->hm.s.u32HMError = uVector;
9324	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9325	}
9326	break;
9327	}
9328	}
9329	break;
9330	}
9331
9332	default:
9333	{
9334	pVCpu->hm.s.u32HMError = uExitIntInfo;
9335	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9336	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9337	break;
9338	}
9339	}
9340	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9341	return rc;
9342	}
9343
9344
9345	/**
9346	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9347	*/
9348	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9349	{
9350	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9351
9352	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9353	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9354	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9355	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9356	AssertRCReturn(rc, rc);
9357
9358	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9359	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9360	return VINF_SUCCESS;
9361	}
9362
9363
9364	/**
9365	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9366	*/
9367	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9368	{
9369	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9370	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9371	HMVMX_RETURN_UNEXPECTED_EXIT();
9372	}
9373
9374
9375	/**
9376	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9377	*/
9378	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9379	{
9380	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9381	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9382	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9383	}
9384
9385
9386	/**
9387	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9388	*/
9389	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9390	{
9391	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9392	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9393	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9394	}
9395
9396
9397	/**
9398	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9399	*/
9400	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9401	{
9402	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9403	PVM pVM = pVCpu->CTX_SUFF(pVM);
9404	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9405	if (RT_LIKELY(rc == VINF_SUCCESS))
9406	{
9407	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9408	Assert(pVmxTransient->cbInstr == 2);
9409	}
9410	else
9411	{
9412	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9413	rc = VERR_EM_INTERPRETER;
9414	}
9415	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9416	return rc;
9417	}
9418
9419
9420	/**
9421	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9422	*/
9423	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9424	{
9425	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9426	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9427	AssertRCReturn(rc, rc);
9428
9429	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9430	return VINF_EM_RAW_EMULATE_INSTR;
9431
9432	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9433	HMVMX_RETURN_UNEXPECTED_EXIT();
9434	}
9435
9436
9437	/**
9438	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9439	*/
9440	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9441	{
9442	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9443	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9444	AssertRCReturn(rc, rc);
9445
9446	PVM pVM = pVCpu->CTX_SUFF(pVM);
9447	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9448	if (RT_LIKELY(rc == VINF_SUCCESS))
9449	{
9450	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9451	Assert(pVmxTransient->cbInstr == 2);
9452	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9453	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9454	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9455	}
9456	else
9457	{
9458	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9459	rc = VERR_EM_INTERPRETER;
9460	}
9461	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9462	return rc;
9463	}
9464
9465
9466	/**
9467	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9468	*/
9469	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9470	{
9471	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9472	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9473	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9474	AssertRCReturn(rc, rc);
9475
9476	PVM pVM = pVCpu->CTX_SUFF(pVM);
9477	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9478	if (RT_LIKELY(rc == VINF_SUCCESS))
9479	{
9480	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9481	Assert(pVmxTransient->cbInstr == 3);
9482	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9483	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9484	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9485	}
9486	else
9487	{
9488	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9489	rc = VERR_EM_INTERPRETER;
9490	}
9491	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9492	return rc;
9493	}
9494
9495
9496	/**
9497	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9498	*/
9499	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9500	{
9501	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9502	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9503	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9504	AssertRCReturn(rc, rc);
9505
9506	PVM pVM = pVCpu->CTX_SUFF(pVM);
9507	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9508	if (RT_LIKELY(rc == VINF_SUCCESS))
9509	{
9510	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9511	Assert(pVmxTransient->cbInstr == 2);
9512	}
9513	else
9514	{
9515	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9516	rc = VERR_EM_INTERPRETER;
9517	}
9518	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9519	return rc;
9520	}
9521
9522
9523	/**
9524	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9525	*/
9526	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9527	{
9528	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9529	PVM pVM = pVCpu->CTX_SUFF(pVM);
9530	Assert(!pVM->hm.s.fNestedPaging);
9531
9532	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9533	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9534	AssertRCReturn(rc, rc);
9535
9536	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9537	rc = VBOXSTRICTRC_VAL(rc2);
9538	if (RT_LIKELY(rc == VINF_SUCCESS))
9539	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9540	else
9541	{
9542	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9543	pVmxTransient->uExitQualification, rc));
9544	}
9545	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9546	return rc;
9547	}
9548
9549
9550	/**
9551	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9552	*/
9553	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9554	{
9555	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9556	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9557	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9558	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9559	AssertRCReturn(rc, rc);
9560
9561	PVM pVM = pVCpu->CTX_SUFF(pVM);
9562	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9563	if (RT_LIKELY(rc == VINF_SUCCESS))
9564	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9565	else
9566	{
9567	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9568	rc = VERR_EM_INTERPRETER;
9569	}
9570	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9571	return rc;
9572	}
9573
9574
9575	/**
9576	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9577	*/
9578	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9579	{
9580	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9581	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9582	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9583	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9584	AssertRCReturn(rc, rc);
9585
9586	PVM pVM = pVCpu->CTX_SUFF(pVM);
9587	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9588	rc = VBOXSTRICTRC_VAL(rc2);
9589	if (RT_LIKELY( rc == VINF_SUCCESS
9590	\|\| rc == VINF_EM_HALT))
9591	{
9592	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9593	AssertRCReturn(rc3, rc3);
9594
9595	if ( rc == VINF_EM_HALT
9596	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9597	{
9598	rc = VINF_SUCCESS;
9599	}
9600	}
9601	else
9602	{
9603	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9604	rc = VERR_EM_INTERPRETER;
9605	}
9606	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9607	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9608	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9609	return rc;
9610	}
9611
9612
9613	/**
9614	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9615	*/
9616	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9617	{
9618	/*
9619	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9620	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9621	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9622	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9623	*/
9624	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9625	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9626	HMVMX_RETURN_UNEXPECTED_EXIT();
9627	}
9628
9629
9630	/**
9631	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9632	*/
9633	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9634	{
9635	/*
9636	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9637	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9638	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9639	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9640	*/
9641	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9642	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9643	HMVMX_RETURN_UNEXPECTED_EXIT();
9644	}
9645
9646
9647	/**
9648	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9649	*/
9650	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9651	{
9652	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9653	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9654	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9655	HMVMX_RETURN_UNEXPECTED_EXIT();
9656	}
9657
9658
9659	/**
9660	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9661	*/
9662	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9663	{
9664	/*
9665	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9666	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9667	* See Intel spec. 25.3 "Other Causes of VM-exits".
9668	*/
9669	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9670	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9671	HMVMX_RETURN_UNEXPECTED_EXIT();
9672	}
9673
9674
9675	/**
9676	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9677	* VM-exit.
9678	*/
9679	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9680	{
9681	/*
9682	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9683	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9684	*
9685	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9686	* See Intel spec. "23.8 Restrictions on VMX operation".
9687	*/
9688	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9689	return VINF_SUCCESS;
9690	}
9691
9692
9693	/**
9694	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9695	* VM-exit.
9696	*/
9697	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9698	{
9699	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9700	return VINF_EM_RESET;
9701	}
9702
9703
9704	/**
9705	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9706	*/
9707	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9708	{
9709	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9710	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9711	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9712	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9713	AssertRCReturn(rc, rc);
9714
9715	pMixedCtx->rip++;
9716	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9717	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9718	rc = VINF_SUCCESS;
9719	else
9720	rc = VINF_EM_HALT;
9721
9722	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9723	return rc;
9724	}
9725
9726
9727	/**
9728	* VM-exit handler for instructions that result in a #UD exception delivered to
9729	* the guest.
9730	*/
9731	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9732	{
9733	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9734	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9735	return VINF_SUCCESS;
9736	}
9737
9738
9739	/**
9740	* VM-exit handler for expiry of the VMX preemption timer.
9741	*/
9742	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9743	{
9744	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9745
9746	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9747	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9748
9749	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9750	PVM pVM = pVCpu->CTX_SUFF(pVM);
9751	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9752	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9753	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9754	}
9755
9756
9757	/**
9758	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9759	*/
9760	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9761	{
9762	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9763
9764	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9765	/** @todo check if XSETBV is supported by the recompiler. */
9766	return VERR_EM_INTERPRETER;
9767	}
9768
9769
9770	/**
9771	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9772	*/
9773	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9774	{
9775	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9776
9777	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9778	/** @todo implement EMInterpretInvpcid() */
9779	return VERR_EM_INTERPRETER;
9780	}
9781
9782
9783	/**
9784	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9785	* Error VM-exit.
9786	*/
9787	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9788	{
9789	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9790	AssertRCReturn(rc, rc);
9791
9792	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9793	NOREF(uInvalidReason);
9794
9795	#ifdef VBOX_STRICT
9796	uint32_t uIntrState;
9797	HMVMXHCUINTREG uHCReg;
9798	uint64_t u64Val;
9799	uint32_t u32Val;
9800
9801	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9802	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9803	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9804	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9805	AssertRCReturn(rc, rc);
9806
9807	Log4(("uInvalidReason %u\n", uInvalidReason));
9808	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9809	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9810	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9811	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9812
9813	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9814	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9815	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9816	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9817	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9818	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9819	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9820	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9821	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9822	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9823	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9824	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9825	#else
9826	NOREF(pVmxTransient);
9827	#endif
9828
9829	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9830	return VERR_VMX_INVALID_GUEST_STATE;
9831	}
9832
9833
9834	/**
9835	* VM-exit handler for VM-entry failure due to an MSR-load
9836	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9837	*/
9838	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9839	{
9840	NOREF(pVmxTransient);
9841	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9842	HMVMX_RETURN_UNEXPECTED_EXIT();
9843	}
9844
9845
9846	/**
9847	* VM-exit handler for VM-entry failure due to a machine-check event
9848	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9849	*/
9850	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9851	{
9852	NOREF(pVmxTransient);
9853	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9854	HMVMX_RETURN_UNEXPECTED_EXIT();
9855	}
9856
9857
9858	/**
9859	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9860	* theory.
9861	*/
9862	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9863	{
9864	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9865	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9866	return VERR_VMX_UNDEFINED_EXIT_CODE;
9867	}
9868
9869
9870	/**
9871	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9872	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9873	* Conditional VM-exit.
9874	*/
9875	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9876	{
9877	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9878
9879	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9880	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9881	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9882	return VERR_EM_INTERPRETER;
9883	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9884	HMVMX_RETURN_UNEXPECTED_EXIT();
9885	}
9886
9887
9888	/**
9889	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9890	*/
9891	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9892	{
9893	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9894
9895	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9896	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9897	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9898	return VERR_EM_INTERPRETER;
9899	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9900	HMVMX_RETURN_UNEXPECTED_EXIT();
9901	}
9902
9903
9904	/**
9905	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9906	*/
9907	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9908	{
9909	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9910
9911	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9912	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9913	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9914	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9915	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9916	{
9917	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9918	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9919	}
9920	AssertRCReturn(rc, rc);
9921	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9922
9923	#ifdef VBOX_STRICT
9924	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
9925	{
9926	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9927	{
9928	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9929	HMVMX_RETURN_UNEXPECTED_EXIT();
9930	}
9931	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9932	if ( HMVMX_IS_64BIT_HOST_MODE()
9933	&& pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
9934	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
9935	{
9936	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
9937	HMVMX_RETURN_UNEXPECTED_EXIT();
9938	}
9939	# endif
9940	}
9941	#endif
9942
9943	PVM pVM = pVCpu->CTX_SUFF(pVM);
9944	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9945	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9946	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9947	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9948
9949	if (RT_LIKELY(rc == VINF_SUCCESS))
9950	{
9951	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9952	Assert(pVmxTransient->cbInstr == 2);
9953	}
9954	return rc;
9955	}
9956
9957
9958	/**
9959	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9960	*/
9961	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9962	{
9963	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9964	PVM pVM = pVCpu->CTX_SUFF(pVM);
9965	int rc = VINF_SUCCESS;
9966
9967	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9968	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9969	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9970	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9971	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9972	{
9973	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9974	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9975	}
9976	AssertRCReturn(rc, rc);
9977	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9978
9979	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9980	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9981	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9982
9983	if (RT_LIKELY(rc == VINF_SUCCESS))
9984	{
9985	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9986
9987	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9988	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9989	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9990	{
9991	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9992	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9993	EMInterpretWrmsr() changes it. */
9994	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9995	}
9996	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9997	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9998
9999	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10000	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10001	{
10002	switch (pMixedCtx->ecx)
10003	{
10004	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10005	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10006	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10007	case MSR_K8_FS_BASE: /* no break */
10008	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10009	default:
10010	{
10011	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10012	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10013	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
10014	else if ( HMVMX_IS_64BIT_HOST_MODE()
10015	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10016	{
10017	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10018	}
10019	#endif
10020	break;
10021	}
10022	}
10023	}
10024	#ifdef VBOX_STRICT
10025	else
10026	{
10027	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10028	switch (pMixedCtx->ecx)
10029	{
10030	case MSR_IA32_SYSENTER_CS:
10031	case MSR_IA32_SYSENTER_EIP:
10032	case MSR_IA32_SYSENTER_ESP:
10033	case MSR_K8_FS_BASE:
10034	case MSR_K8_GS_BASE:
10035	{
10036	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10037	HMVMX_RETURN_UNEXPECTED_EXIT();
10038	}
10039
10040	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10041	default:
10042	{
10043	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10044	{
10045	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10046	pMixedCtx->ecx));
10047	HMVMX_RETURN_UNEXPECTED_EXIT();
10048	}
10049
10050	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
10051	if ( HMVMX_IS_64BIT_HOST_MODE()
10052	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10053	{
10054	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10055	HMVMX_RETURN_UNEXPECTED_EXIT();
10056	}
10057	#endif
10058	break;
10059	}
10060	}
10061	}
10062	#endif /* VBOX_STRICT */
10063	}
10064	return rc;
10065	}
10066
10067
10068	/**
10069	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10070	*/
10071	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10072	{
10073	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10074
10075	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10076	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10077	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10078	return VERR_EM_INTERPRETER;
10079	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10080	HMVMX_RETURN_UNEXPECTED_EXIT();
10081	}
10082
10083
10084	/**
10085	* VM-exit handler for when the TPR value is lowered below the specified
10086	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10087	*/
10088	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10089	{
10090	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10091	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10092
10093	/*
10094	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10095	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10096	* resume guest execution.
10097	*/
10098	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10099	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10100	return VINF_SUCCESS;
10101	}
10102
10103
10104	/**
10105	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10106	* VM-exit.
10107	*
10108	* @retval VINF_SUCCESS when guest execution can continue.
10109	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10110	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10111	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10112	* recompiler.
10113	*/
10114	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10115	{
10116	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10117	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10118	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10119	AssertRCReturn(rc, rc);
10120
10121	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10122	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10123	PVM pVM = pVCpu->CTX_SUFF(pVM);
10124	switch (uAccessType)
10125	{
10126	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10127	{
10128	#if 0
10129	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10130	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10131	#else
10132	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10133	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10134	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10135	#endif
10136	AssertRCReturn(rc, rc);
10137
10138	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10139	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10140	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10141	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
10142
10143	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10144	{
10145	case 0: /* CR0 */
10146	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10147	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10148	break;
10149	case 2: /* CR2 */
10150	/* Nothing to do here, CR2 it's not part of the VMCS. */
10151	break;
10152	case 3: /* CR3 */
10153	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10154	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10155	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10156	break;
10157	case 4: /* CR4 */
10158	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10159	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10160	break;
10161	case 8: /* CR8 */
10162	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10163	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10164	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10165	break;
10166	default:
10167	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10168	break;
10169	}
10170
10171	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10172	break;
10173	}
10174
10175	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10176	{
10177	/* EMInterpretCRxRead() requires EFER MSR, CS. */
10178	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10179	AssertRCReturn(rc, rc);
10180	Assert( !pVM->hm.s.fNestedPaging
10181	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10182	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10183
10184	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10185	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10186	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10187
10188	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10189	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10190	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10191	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10192	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10193	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10194	break;
10195	}
10196
10197	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10198	{
10199	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10200	AssertRCReturn(rc, rc);
10201	rc = EMInterpretCLTS(pVM, pVCpu);
10202	AssertRCReturn(rc, rc);
10203	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10204	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10205	Log4(("CRX CLTS write rc=%d\n", rc));
10206	break;
10207	}
10208
10209	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10210	{
10211	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10212	AssertRCReturn(rc, rc);
10213	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10214	if (RT_LIKELY(rc == VINF_SUCCESS))
10215	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10216	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10217	Log4(("CRX LMSW write rc=%d\n", rc));
10218	break;
10219	}
10220
10221	default:
10222	{
10223	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10224	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10225	}
10226	}
10227
10228	/* Validate possible error codes. */
10229	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10230	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10231	if (RT_SUCCESS(rc))
10232	{
10233	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10234	AssertRCReturn(rc2, rc2);
10235	}
10236
10237	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10238	return rc;
10239	}
10240
10241
10242	/**
10243	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10244	* VM-exit.
10245	*/
10246	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10247	{
10248	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10249	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10250
10251	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10252	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10253	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10254	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10255	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10256	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10257	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10258	AssertRCReturn(rc2, rc2);
10259
10260	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10261	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10262	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10263	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10264	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10265	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10266	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10267
10268	/* I/O operation lookup arrays. */
10269	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10270	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10271
10272	VBOXSTRICTRC rcStrict;
10273	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10274	const uint32_t cbInstr = pVmxTransient->cbInstr;
10275	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10276	PVM pVM = pVCpu->CTX_SUFF(pVM);
10277	if (fIOString)
10278	{
10279	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10280	/*
10281	* INS/OUTS - I/O String instruction.
10282	*
10283	* Use instruction-information if available, otherwise fall back on
10284	* interpreting the instruction.
10285	*/
10286	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10287	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10288	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10289	{
10290	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10291	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10292	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10293	AssertRCReturn(rc2, rc2);
10294	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10295	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10296	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10297	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10298	if (fIOWrite)
10299	{
10300	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10301	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10302	}
10303	else
10304	{
10305	/*
10306	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10307	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10308	* See Intel Instruction spec. for "INS".
10309	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10310	*/
10311	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10312	}
10313	}
10314	else
10315	{
10316	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10317	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10318	AssertRCReturn(rc2, rc2);
10319	rcStrict = IEMExecOne(pVCpu);
10320	}
10321	/** @todo IEM needs to be setting these flags somehow. */
10322	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10323	fUpdateRipAlready = true;
10324	#else
10325	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10326	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10327	if (RT_SUCCESS(rcStrict))
10328	{
10329	if (fIOWrite)
10330	{
10331	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10332	(DISCPUMODE)pDis->uAddrMode, cbValue);
10333	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10334	}
10335	else
10336	{
10337	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10338	(DISCPUMODE)pDis->uAddrMode, cbValue);
10339	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10340	}
10341	}
10342	else
10343	{
10344	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10345	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10346	}
10347	#endif
10348	}
10349	else
10350	{
10351	/*
10352	* IN/OUT - I/O instruction.
10353	*/
10354	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10355	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10356	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10357	if (fIOWrite)
10358	{
10359	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10360	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10361	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10362	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10363	}
10364	else
10365	{
10366	uint32_t u32Result = 0;
10367	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10368	if (IOM_SUCCESS(rcStrict))
10369	{
10370	/* Save result of I/O IN instr. in AL/AX/EAX. */
10371	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10372	}
10373	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10374	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10375	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10376	}
10377	}
10378
10379	if (IOM_SUCCESS(rcStrict))
10380	{
10381	if (!fUpdateRipAlready)
10382	{
10383	pMixedCtx->rip += cbInstr;
10384	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10385	}
10386
10387	/*
10388	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10389	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10390	*/
10391	if (fIOString)
10392	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10393
10394	/*
10395	* If any I/O breakpoints are armed, we need to check if one triggered
10396	* and take appropriate action.
10397	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10398	*/
10399	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10400	AssertRCReturn(rc2, rc2);
10401
10402	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10403	* execution engines about whether hyper BPs and such are pending. */
10404	uint32_t const uDr7 = pMixedCtx->dr[7];
10405	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10406	&& X86_DR7_ANY_RW_IO(uDr7)
10407	&& (pMixedCtx->cr4 & X86_CR4_DE))
10408	\|\| DBGFBpIsHwIoArmed(pVM)))
10409	{
10410	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10411
10412	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10413	VMMRZCallRing3Disable(pVCpu);
10414	HM_DISABLE_PREEMPT_IF_NEEDED();
10415
10416	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10417
10418	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10419	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10420	{
10421	/* Raise #DB. */
10422	if (fIsGuestDbgActive)
10423	ASMSetDR6(pMixedCtx->dr[6]);
10424	if (pMixedCtx->dr[7] != uDr7)
10425	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10426
10427	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10428	}
10429	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10430	else if ( rcStrict2 != VINF_SUCCESS
10431	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10432	rcStrict = rcStrict2;
10433
10434	HM_RESTORE_PREEMPT_IF_NEEDED();
10435	VMMRZCallRing3Enable(pVCpu);
10436	}
10437	}
10438
10439	#ifdef DEBUG
10440	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10441	Assert(!fIOWrite);
10442	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10443	Assert(fIOWrite);
10444	else
10445	{
10446	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10447	* statuses, that the VMM device and some others may return. See
10448	* IOM_SUCCESS() for guidance. */
10449	AssertMsg( RT_FAILURE(rcStrict)
10450	\|\| rcStrict == VINF_SUCCESS
10451	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10452	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10453	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10454	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10455	}
10456	#endif
10457
10458	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10459	return VBOXSTRICTRC_TODO(rcStrict);
10460	}
10461
10462
10463	/**
10464	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10465	* VM-exit.
10466	*/
10467	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10468	{
10469	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10470
10471	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10472	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10473	AssertRCReturn(rc, rc);
10474	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10475	{
10476	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10477	AssertRCReturn(rc, rc);
10478	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10479	{
10480	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10481
10482	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10483	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10484	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10485	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10486	{
10487	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10488	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10489
10490	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10491	Assert(!pVCpu->hm.s.Event.fPending);
10492	pVCpu->hm.s.Event.fPending = true;
10493	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10494	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10495	AssertRCReturn(rc, rc);
10496	if (fErrorCodeValid)
10497	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10498	else
10499	pVCpu->hm.s.Event.u32ErrCode = 0;
10500	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10501	&& uVector == X86_XCPT_PF)
10502	{
10503	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10504	}
10505
10506	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10507	}
10508	}
10509	}
10510
10511	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10512	* emulation. */
10513	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10514	return VERR_EM_INTERPRETER;
10515	}
10516
10517
10518	/**
10519	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10520	*/
10521	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10522	{
10523	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10524	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10525	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10526	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10527	AssertRCReturn(rc, rc);
10528	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10529	return VINF_EM_DBG_STEPPED;
10530	}
10531
10532
10533	/**
10534	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10535	*/
10536	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10537	{
10538	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10539
10540	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10541	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10542	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10543	return VINF_SUCCESS;
10544	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10545	return rc;
10546
10547	#if 0
10548	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10549	* just sync the whole thing. */
10550	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10551	#else
10552	/* Aggressive state sync. for now. */
10553	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10554	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10555	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10556	#endif
10557	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10558	AssertRCReturn(rc, rc);
10559
10560	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10561	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10562	switch (uAccessType)
10563	{
10564	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10565	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10566	{
10567	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10568	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10569	{
10570	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10571	}
10572
10573	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10574	GCPhys &= PAGE_BASE_GC_MASK;
10575	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10576	PVM pVM = pVCpu->CTX_SUFF(pVM);
10577	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10578	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10579
10580	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10581	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10582	CPUMCTX2CORE(pMixedCtx), GCPhys);
10583	rc = VBOXSTRICTRC_VAL(rc2);
10584	Log4(("ApicAccess rc=%d\n", rc));
10585	if ( rc == VINF_SUCCESS
10586	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10587	\|\| rc == VERR_PAGE_NOT_PRESENT)
10588	{
10589	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10590	\| HM_CHANGED_GUEST_RSP
10591	\| HM_CHANGED_GUEST_RFLAGS
10592	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10593	rc = VINF_SUCCESS;
10594	}
10595	break;
10596	}
10597
10598	default:
10599	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10600	rc = VINF_EM_RAW_EMULATE_INSTR;
10601	break;
10602	}
10603
10604	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10605	return rc;
10606	}
10607
10608
10609	/**
10610	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10611	* VM-exit.
10612	*/
10613	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10614	{
10615	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10616
10617	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10618	if (pVmxTransient->fWasGuestDebugStateActive)
10619	{
10620	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10621	HMVMX_RETURN_UNEXPECTED_EXIT();
10622	}
10623
10624	int rc = VERR_INTERNAL_ERROR_5;
10625	if ( !DBGFIsStepping(pVCpu)
10626	&& !pVCpu->hm.s.fSingleInstruction
10627	&& !pVmxTransient->fWasHyperDebugStateActive)
10628	{
10629	/* Don't intercept MOV DRx and #DB any more. */
10630	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10631	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10632	AssertRCReturn(rc, rc);
10633
10634	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10635	{
10636	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10637	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10638	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10639	AssertRCReturn(rc, rc);
10640	#endif
10641	}
10642
10643	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10644	VMMRZCallRing3Disable(pVCpu);
10645	HM_DISABLE_PREEMPT_IF_NEEDED();
10646
10647	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10648	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10649	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10650
10651	HM_RESTORE_PREEMPT_IF_NEEDED();
10652	VMMRZCallRing3Enable(pVCpu);
10653
10654	#ifdef VBOX_WITH_STATISTICS
10655	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10656	AssertRCReturn(rc, rc);
10657	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10658	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10659	else
10660	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10661	#endif
10662	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10663	return VINF_SUCCESS;
10664	}
10665
10666	/*
10667	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10668	* Update the segment registers and DR7 from the CPU.
10669	*/
10670	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10671	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10672	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10673	AssertRCReturn(rc, rc);
10674	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10675
10676	PVM pVM = pVCpu->CTX_SUFF(pVM);
10677	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10678	{
10679	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10680	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10681	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10682	if (RT_SUCCESS(rc))
10683	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10684	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10685	}
10686	else
10687	{
10688	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10689	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10690	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10691	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10692	}
10693
10694	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10695	if (RT_SUCCESS(rc))
10696	{
10697	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10698	AssertRCReturn(rc2, rc2);
10699	}
10700	return rc;
10701	}
10702
10703
10704	/**
10705	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10706	* Conditional VM-exit.
10707	*/
10708	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10709	{
10710	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10711	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10712
10713	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10714	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10715	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10716	return VINF_SUCCESS;
10717	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10718	return rc;
10719
10720	RTGCPHYS GCPhys = 0;
10721	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10722
10723	#if 0
10724	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10725	#else
10726	/* Aggressive state sync. for now. */
10727	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10728	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10729	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10730	#endif
10731	AssertRCReturn(rc, rc);
10732
10733	/*
10734	* If we succeed, resume guest execution.
10735	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10736	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10737	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10738	* weird case. See @bugref{6043}.
10739	*/
10740	PVM pVM = pVCpu->CTX_SUFF(pVM);
10741	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10742	rc = VBOXSTRICTRC_VAL(rc2);
10743	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10744	if ( rc == VINF_SUCCESS
10745	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10746	\|\| rc == VERR_PAGE_NOT_PRESENT)
10747	{
10748	/* Successfully handled MMIO operation. */
10749	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10750	\| HM_CHANGED_GUEST_RSP
10751	\| HM_CHANGED_GUEST_RFLAGS
10752	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10753	rc = VINF_SUCCESS;
10754	}
10755	return rc;
10756	}
10757
10758
10759	/**
10760	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10761	* VM-exit.
10762	*/
10763	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10764	{
10765	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10766	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10767
10768	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10769	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10770	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10771	return VINF_SUCCESS;
10772	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10773	return rc;
10774
10775	RTGCPHYS GCPhys = 0;
10776	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10777	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10778	#if 0
10779	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10780	#else
10781	/* Aggressive state sync. for now. */
10782	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10783	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10784	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10785	#endif
10786	AssertRCReturn(rc, rc);
10787
10788	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10789	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10790
10791	RTGCUINT uErrorCode = 0;
10792	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10793	uErrorCode \|= X86_TRAP_PF_ID;
10794	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10795	uErrorCode \|= X86_TRAP_PF_RW;
10796	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10797	uErrorCode \|= X86_TRAP_PF_P;
10798
10799	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10800
10801	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10802	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10803
10804	/* Handle the pagefault trap for the nested shadow table. */
10805	PVM pVM = pVCpu->CTX_SUFF(pVM);
10806	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10807	TRPMResetTrap(pVCpu);
10808
10809	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10810	if ( rc == VINF_SUCCESS
10811	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10812	\|\| rc == VERR_PAGE_NOT_PRESENT)
10813	{
10814	/* Successfully synced our nested page tables. */
10815	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10816	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10817	\| HM_CHANGED_GUEST_RSP
10818	\| HM_CHANGED_GUEST_RFLAGS);
10819	return VINF_SUCCESS;
10820	}
10821
10822	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10823	return rc;
10824	}
10825
10826	/** @} */
10827
10828	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10829	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10830	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10831
10832	/** @name VM-exit exception handlers.
10833	* @{
10834	*/
10835
10836	/**
10837	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10838	*/
10839	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10840	{
10841	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10842	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10843
10844	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10845	AssertRCReturn(rc, rc);
10846
10847	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10848	{
10849	/* Convert a #MF into a FERR -> IRQ 13. */
10850	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /uTagSrc/);
10851	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10852	AssertRCReturn(rc2, rc2);
10853	return rc;
10854	}
10855
10856	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10857	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10858	return rc;
10859	}
10860
10861
10862	/**
10863	* VM-exit exception handler for #BP (Breakpoint exception).
10864	*/
10865	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10866	{
10867	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10868	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10869
10870	/** @todo Try optimize this by not saving the entire guest state unless
10871	* really needed. */
10872	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10873	AssertRCReturn(rc, rc);
10874
10875	PVM pVM = pVCpu->CTX_SUFF(pVM);
10876	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10877	if (rc == VINF_EM_RAW_GUEST_TRAP)
10878	{
10879	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10880	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10881	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10882	AssertRCReturn(rc, rc);
10883
10884	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10885	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10886	}
10887
10888	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10889	return rc;
10890	}
10891
10892
10893	/**
10894	* VM-exit exception handler for #DB (Debug exception).
10895	*/
10896	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10897	{
10898	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10899	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10900	Log6(("XcptDB\n"));
10901
10902	/*
10903	* Get the DR6-like values from the exit qualification and pass it to DBGF
10904	* for processing.
10905	*/
10906	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10907	AssertRCReturn(rc, rc);
10908
10909	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10910	uint64_t uDR6 = X86_DR6_INIT_VAL;
10911	uDR6 \|= ( pVmxTransient->uExitQualification
10912	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10913
10914	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10915	if (rc == VINF_EM_RAW_GUEST_TRAP)
10916	{
10917	/*
10918	* The exception was for the guest. Update DR6, DR7.GD and
10919	* IA32_DEBUGCTL.LBR before forwarding it.
10920	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10921	*/
10922	VMMRZCallRing3Disable(pVCpu);
10923	HM_DISABLE_PREEMPT_IF_NEEDED();
10924
10925	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10926	pMixedCtx->dr[6] \|= uDR6;
10927	if (CPUMIsGuestDebugStateActive(pVCpu))
10928	ASMSetDR6(pMixedCtx->dr[6]);
10929
10930	HM_RESTORE_PREEMPT_IF_NEEDED();
10931	VMMRZCallRing3Enable(pVCpu);
10932
10933	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10934	AssertRCReturn(rc, rc);
10935
10936	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10937	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10938
10939	/* Paranoia. */
10940	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10941	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10942
10943	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10944	AssertRCReturn(rc, rc);
10945
10946	/*
10947	* Raise #DB in the guest.
10948	*/
10949	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10950	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10951	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10952	AssertRCReturn(rc, rc);
10953	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10954	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10955	return VINF_SUCCESS;
10956	}
10957
10958	/*
10959	* Not a guest trap, must be a hypervisor related debug event then.
10960	* Update DR6 in case someone is interested in it.
10961	*/
10962	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10963	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10964	CPUMSetHyperDR6(pVCpu, uDR6);
10965
10966	return rc;
10967	}
10968
10969
10970	/**
10971	* VM-exit exception handler for #NM (Device-not-available exception: floating
10972	* point exception).
10973	*/
10974	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10975	{
10976	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10977
10978	/* We require CR0 and EFER. EFER is always up-to-date. */
10979	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10980	AssertRCReturn(rc, rc);
10981
10982	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10983	VMMRZCallRing3Disable(pVCpu);
10984	HM_DISABLE_PREEMPT_IF_NEEDED();
10985
10986	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10987	if (pVmxTransient->fWasGuestFPUStateActive)
10988	{
10989	rc = VINF_EM_RAW_GUEST_TRAP;
10990	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10991	}
10992	else
10993	{
10994	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10995	Assert(!pVmxTransient->fWasGuestFPUStateActive);
10996	#endif
10997	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10998	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10999	}
11000
11001	HM_RESTORE_PREEMPT_IF_NEEDED();
11002	VMMRZCallRing3Enable(pVCpu);
11003
11004	if (rc == VINF_SUCCESS)
11005	{
11006	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11007	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11008	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11009	pVCpu->hm.s.fUseGuestFpu = true;
11010	}
11011	else
11012	{
11013	/* Forward #NM to the guest. */
11014	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11015	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11016	AssertRCReturn(rc, rc);
11017	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11018	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11019	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11020	}
11021
11022	return VINF_SUCCESS;
11023	}
11024
11025
11026	/**
11027	* VM-exit exception handler for #GP (General-protection exception).
11028	*
11029	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11030	*/
11031	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11032	{
11033	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11034	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11035
11036	int rc = VERR_INTERNAL_ERROR_5;
11037	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11038	{
11039	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11040	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11041	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11042	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11043	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11044	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11045	AssertRCReturn(rc, rc);
11046	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11047	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11048	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11049	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11050	return rc;
11051	#else
11052	/* We don't intercept #GP. */
11053	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11054	NOREF(pVmxTransient);
11055	return VERR_VMX_UNEXPECTED_EXCEPTION;
11056	#endif
11057	}
11058
11059	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11060	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11061
11062	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11063	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11064	AssertRCReturn(rc, rc);
11065
11066	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11067	uint32_t cbOp = 0;
11068	PVM pVM = pVCpu->CTX_SUFF(pVM);
11069	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11070	if (RT_SUCCESS(rc))
11071	{
11072	rc = VINF_SUCCESS;
11073	Assert(cbOp == pDis->cbInstr);
11074	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11075	switch (pDis->pCurInstr->uOpcode)
11076	{
11077	case OP_CLI:
11078	{
11079	pMixedCtx->eflags.Bits.u1IF = 0;
11080	pMixedCtx->rip += pDis->cbInstr;
11081	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11082	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11083	break;
11084	}
11085
11086	case OP_STI:
11087	{
11088	pMixedCtx->eflags.Bits.u1IF = 1;
11089	pMixedCtx->rip += pDis->cbInstr;
11090	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11091	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11092	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11093	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11094	break;
11095	}
11096
11097	case OP_HLT:
11098	{
11099	rc = VINF_EM_HALT;
11100	pMixedCtx->rip += pDis->cbInstr;
11101	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11102	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11103	break;
11104	}
11105
11106	case OP_POPF:
11107	{
11108	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11109	uint32_t cbParm = 0;
11110	uint32_t uMask = 0;
11111	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11112	{
11113	cbParm = 4;
11114	uMask = 0xffffffff;
11115	}
11116	else
11117	{
11118	cbParm = 2;
11119	uMask = 0xffff;
11120	}
11121
11122	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11123	RTGCPTR GCPtrStack = 0;
11124	X86EFLAGS Eflags;
11125	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11126	&GCPtrStack);
11127	if (RT_SUCCESS(rc))
11128	{
11129	Assert(sizeof(Eflags.u32) >= cbParm);
11130	Eflags.u32 = 0;
11131	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11132	}
11133	if (RT_FAILURE(rc))
11134	{
11135	rc = VERR_EM_INTERPRETER;
11136	break;
11137	}
11138	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11139	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11140	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11141	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11142	pMixedCtx->esp += cbParm;
11143	pMixedCtx->esp &= uMask;
11144	pMixedCtx->rip += pDis->cbInstr;
11145
11146	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11147	\| HM_CHANGED_GUEST_RSP
11148	\| HM_CHANGED_GUEST_RFLAGS);
11149	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11150	break;
11151	}
11152
11153	case OP_PUSHF:
11154	{
11155	uint32_t cbParm = 0;
11156	uint32_t uMask = 0;
11157	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11158	{
11159	cbParm = 4;
11160	uMask = 0xffffffff;
11161	}
11162	else
11163	{
11164	cbParm = 2;
11165	uMask = 0xffff;
11166	}
11167
11168	/* Get the stack pointer & push the contents of eflags onto the stack. */
11169	RTGCPTR GCPtrStack = 0;
11170	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11171	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11172	if (RT_FAILURE(rc))
11173	{
11174	rc = VERR_EM_INTERPRETER;
11175	break;
11176	}
11177	X86EFLAGS Eflags = pMixedCtx->eflags;
11178	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11179	Eflags.Bits.u1RF = 0;
11180	Eflags.Bits.u1VM = 0;
11181
11182	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11183	if (RT_FAILURE(rc))
11184	{
11185	rc = VERR_EM_INTERPRETER;
11186	break;
11187	}
11188	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11189	pMixedCtx->esp -= cbParm;
11190	pMixedCtx->esp &= uMask;
11191	pMixedCtx->rip += pDis->cbInstr;
11192	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
11193	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11194	break;
11195	}
11196
11197	case OP_IRET:
11198	{
11199	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11200	* instruction reference. */
11201	RTGCPTR GCPtrStack = 0;
11202	uint32_t uMask = 0xffff;
11203	uint16_t aIretFrame[3];
11204	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11205	{
11206	rc = VERR_EM_INTERPRETER;
11207	break;
11208	}
11209	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11210	&GCPtrStack);
11211	if (RT_SUCCESS(rc))
11212	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11213	if (RT_FAILURE(rc))
11214	{
11215	rc = VERR_EM_INTERPRETER;
11216	break;
11217	}
11218	pMixedCtx->eip = 0;
11219	pMixedCtx->ip = aIretFrame[0];
11220	pMixedCtx->cs.Sel = aIretFrame[1];
11221	pMixedCtx->cs.ValidSel = aIretFrame[1];
11222	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11223	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11224	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11225	pMixedCtx->sp += sizeof(aIretFrame);
11226	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11227	\| HM_CHANGED_GUEST_SEGMENT_REGS
11228	\| HM_CHANGED_GUEST_RSP
11229	\| HM_CHANGED_GUEST_RFLAGS);
11230	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11232	break;
11233	}
11234
11235	case OP_INT:
11236	{
11237	uint16_t uVector = pDis->Param1.uValue & 0xff;
11238	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11239	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11240	break;
11241	}
11242
11243	case OP_INTO:
11244	{
11245	if (pMixedCtx->eflags.Bits.u1OF)
11246	{
11247	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11248	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11249	}
11250	break;
11251	}
11252
11253	default:
11254	{
11255	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11256	EMCODETYPE_SUPERVISOR);
11257	rc = VBOXSTRICTRC_VAL(rc2);
11258	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11259	Log4(("#GP rc=%Rrc\n", rc));
11260	break;
11261	}
11262	}
11263	}
11264	else
11265	rc = VERR_EM_INTERPRETER;
11266
11267	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11268	("#GP Unexpected rc=%Rrc\n", rc));
11269	return rc;
11270	}
11271
11272
11273	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11274	/**
11275	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11276	* the exception reported in the VMX transient structure back into the VM.
11277	*
11278	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11279	* up-to-date.
11280	*/
11281	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11282	{
11283	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11284
11285	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11286	hmR0VmxCheckExitDueToEventDelivery(). */
11287	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11288	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11289	AssertRCReturn(rc, rc);
11290	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11291
11292	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11293	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11294	return VINF_SUCCESS;
11295	}
11296	#endif
11297
11298
11299	/**
11300	* VM-exit exception handler for #PF (Page-fault exception).
11301	*/
11302	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11303	{
11304	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11305	PVM pVM = pVCpu->CTX_SUFF(pVM);
11306	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11307	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11308	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11309	AssertRCReturn(rc, rc);
11310
11311	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11312	if (pVM->hm.s.fNestedPaging)
11313	{
11314	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11315	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11316	{
11317	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11318	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11319	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11320	}
11321	else
11322	{
11323	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11324	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11325	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11326	}
11327	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11328	return rc;
11329	}
11330	#else
11331	Assert(!pVM->hm.s.fNestedPaging);
11332	NOREF(pVM);
11333	#endif
11334
11335	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11336	AssertRCReturn(rc, rc);
11337
11338	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11339	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11340
11341	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11342	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11343	(RTGCPTR)pVmxTransient->uExitQualification);
11344
11345	Log4(("#PF: rc=%Rrc\n", rc));
11346	if (rc == VINF_SUCCESS)
11347	{
11348	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11349	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11350	* memory? We don't update the whole state here... */
11351	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11352	\| HM_CHANGED_GUEST_RSP
11353	\| HM_CHANGED_GUEST_RFLAGS
11354	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11355	TRPMResetTrap(pVCpu);
11356	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11357	return rc;
11358	}
11359	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11360	{
11361	if (!pVmxTransient->fVectoringPF)
11362	{
11363	/* It's a guest page fault and needs to be reflected to the guest. */
11364	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11365	TRPMResetTrap(pVCpu);
11366	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11367	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11368	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11369	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11370	}
11371	else
11372	{
11373	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11374	TRPMResetTrap(pVCpu);
11375	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11376	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11377	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11378	}
11379
11380	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11381	return VINF_SUCCESS;
11382	}
11383
11384	TRPMResetTrap(pVCpu);
11385	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11386	return rc;
11387	}
11388
11389	/** @} */
11390

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49729

Download in other formats: