HMVMXR0.cpp@ 47389

Last change on this file since 47389 was 47378, checked in by vboxsync, 12 years ago
VMM/HMVMXR0: Typos and make re-checking of CR4.VMXE on VMXR0Enter() only for strict builds.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 390.6 KB

Line
1	/* $Id: HMVMXR0.cpp 47378 2013-07-24 16:32:46Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#if defined(RT_ARCH_AMD64)
49	# define HMVMX_IS_64BIT_HOST_MODE() (true)
50	typedef RTHCUINTREG HMVMXHCUINTREG;
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	typedef uint64_t HMVMXHCUINTREG;
55	#else
56	# define HMVMX_IS_64BIT_HOST_MODE() (false)
57	typedef RTHCUINTREG HMVMXHCUINTREG;
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** Determine which tagged-TLB flush handler to use. */
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69	/** @name Updated-guest-state flags.
70	* @{ */
71	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
83	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
84	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
86	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
88	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
89	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
90	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92	\| HMVMX_UPDATED_GUEST_RSP \
93	\| HMVMX_UPDATED_GUEST_RFLAGS \
94	\| HMVMX_UPDATED_GUEST_CR0 \
95	\| HMVMX_UPDATED_GUEST_CR3 \
96	\| HMVMX_UPDATED_GUEST_CR4 \
97	\| HMVMX_UPDATED_GUEST_GDTR \
98	\| HMVMX_UPDATED_GUEST_IDTR \
99	\| HMVMX_UPDATED_GUEST_LDTR \
100	\| HMVMX_UPDATED_GUEST_TR \
101	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102	\| HMVMX_UPDATED_GUEST_DEBUG \
103	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
104	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
106	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
108	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
109	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
110	\| HMVMX_UPDATED_GUEST_APIC_STATE)
111	/** @} */
112
113	/**
114	* Flags to skip redundant reads of some common VMCS fields that are not part of
115	* the guest-CPU state but are in the transient structure.
116	*/
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123
124	/**
125	* @name Exception bitmap mask for real-mode guests (real-on-v86).
126	*
127	* We need to intercept all exceptions manually (except #PF). #NM is also
128	* handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
129	* intercepted even in real-mode if we have Nested Paging support.
130	*/
131	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
132	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
133	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
134	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
135	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
136	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
137	\| RT_BIT(X86_XCPT_XF))
138	/** @} */
139
140	/**
141	* @name Exception bitmap mask for all contributory exceptions.
142	*
143	* Page fault is deliberately excluded here as it's conditional as to whether
144	* it's contributory or benign. Page faults are handled separately.
145	*/
146	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
147	\| RT_BIT(X86_XCPT_DE))
148	/** @} */
149
150	/** Maximum VM-instruction error number. */
151	#define HMVMX_INSTR_ERROR_MAX 28
152
153	/** Profiling macro. */
154	#ifdef HM_PROFILE_EXIT_DISPATCH
155	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
156	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
157	#else
158	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
159	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
160	#endif
161
162
163	/*******************************************************************************
164	* Structures and Typedefs *
165	*******************************************************************************/
166	/** @name VMX transient.
167	*
168	* A state structure for holding miscellaneous information across
169	* VMX non-root operation and restored after the transition.
170	*
171	* @{ */
172	typedef struct VMXTRANSIENT
173	{
174	/** The host's rflags/eflags. */
175	RTCCUINTREG uEFlags;
176	#if HC_ARCH_BITS == 32
177	uint32_t u32Alignment0;
178	#endif
179	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
180	uint64_t u64LStarMsr;
181	/** The guest's TPR value used for TPR shadowing. */
182	uint8_t u8GuestTpr;
183	/** Alignment. */
184	uint8_t abAlignment0[7];
185
186	/** The basic VM-exit reason. */
187	uint16_t uExitReason;
188	/** Alignment. */
189	uint16_t u16Alignment0;
190	/** The VM-exit interruption error code. */
191	uint32_t uExitIntrErrorCode;
192	/** The VM-exit exit qualification. */
193	uint64_t uExitQualification;
194
195	/** The VM-exit interruption-information field. */
196	uint32_t uExitIntrInfo;
197	/** The VM-exit instruction-length field. */
198	uint32_t cbInstr;
199	/** Whether the VM-entry failed or not. */
200	bool fVMEntryFailed;
201	/** Alignment. */
202	uint8_t abAlignment1[7];
203
204	/** The VM-entry interruption-information field. */
205	uint32_t uEntryIntrInfo;
206	/** The VM-entry exception error code field. */
207	uint32_t uEntryXcptErrorCode;
208	/** The VM-entry instruction length field. */
209	uint32_t cbEntryInstr;
210
211	/** IDT-vectoring information field. */
212	uint32_t uIdtVectoringInfo;
213	/** IDT-vectoring error code. */
214	uint32_t uIdtVectoringErrorCode;
215
216	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
217	uint32_t fVmcsFieldsRead;
218	/** Whether TSC-offsetting should be setup before VM-entry. */
219	bool fUpdateTscOffsettingAndPreemptTimer;
220	/** Whether the VM-exit was caused by a page-fault during delivery of a
221	* contributory exception or a page-fault. */
222	bool fVectoringPF;
223	} VMXTRANSIENT, *PVMXTRANSIENT;
224	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
225	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
226	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
227	/** @} */
228
229
230	/**
231	* MSR-bitmap read permissions.
232	*/
233	typedef enum VMXMSREXITREAD
234	{
235	/** Reading this MSR causes a VM-exit. */
236	VMXMSREXIT_INTERCEPT_READ = 0xb,
237	/** Reading this MSR does not cause a VM-exit. */
238	VMXMSREXIT_PASSTHRU_READ
239	} VMXMSREXITREAD;
240
241	/**
242	* MSR-bitmap write permissions.
243	*/
244	typedef enum VMXMSREXITWRITE
245	{
246	/** Writing to this MSR causes a VM-exit. */
247	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
248	/** Writing to this MSR does not cause a VM-exit. */
249	VMXMSREXIT_PASSTHRU_WRITE
250	} VMXMSREXITWRITE;
251
252
253	/*******************************************************************************
254	* Internal Functions *
255	*******************************************************************************/
256	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
257	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
258	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
259	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
260	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
261	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
262	#endif
263	#ifndef HMVMX_USE_FUNCTION_TABLE
264	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
265	#define HMVMX_EXIT_DECL static int
266	#else
267	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
268	#endif
269
270	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
307	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
309	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314
315	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
316	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
317	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
318	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
319	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
320	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
321	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
322
323
324	/*******************************************************************************
325	* Global Variables *
326	*******************************************************************************/
327	#ifdef HMVMX_USE_FUNCTION_TABLE
328	/**
329	* VM-exit handler.
330	*
331	* @returns VBox status code.
332	* @param pVCpu Pointer to the VMCPU.
333	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
334	* out-of-sync. Make sure to update the required
335	* fields before using them.
336	* @param pVmxTransient Pointer to the VMX-transient structure.
337	*/
338	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
339	/** Pointer to VM-exit handler. */
340	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
341
342	/**
343	* VMX_EXIT dispatch table.
344	*/
345	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
346	{
347	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
348	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
349	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
350	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
351	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
352	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
353	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
354	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
355	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
356	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
357	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
358	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
359	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
360	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
361	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
362	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
363	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
364	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
365	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
366	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
367	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
368	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
369	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
370	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
371	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
372	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
373	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
374	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
375	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
376	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
377	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
378	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
379	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
380	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
381	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
382	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
383	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
384	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
385	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
386	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
387	/* 40 UNDEFINED */ hmR0VmxExitPause,
388	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
389	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
390	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
391	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
392	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
393	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
394	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
395	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
396	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
397	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
398	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
399	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
400	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
401	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
402	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
403	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
404	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
405	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
406	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
407	};
408	#endif /* HMVMX_USE_FUNCTION_TABLE */
409
410	#ifdef VBOX_STRICT
411	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
412	{
413	/* 0 */ "(Not Used)",
414	/* 1 */ "VMCALL executed in VMX root operation.",
415	/* 2 */ "VMCLEAR with invalid physical address.",
416	/* 3 */ "VMCLEAR with VMXON pointer.",
417	/* 4 */ "VMLAUNCH with non-clear VMCS.",
418	/* 5 */ "VMRESUME with non-launched VMCS.",
419	/* 6 */ "VMRESUME after VMXOFF",
420	/* 7 */ "VM entry with invalid control fields.",
421	/* 8 */ "VM entry with invalid host state fields.",
422	/* 9 */ "VMPTRLD with invalid physical address.",
423	/* 10 */ "VMPTRLD with VMXON pointer.",
424	/* 11 */ "VMPTRLD with incorrect revision identifier.",
425	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
426	/* 13 */ "VMWRITE to read-only VMCS component.",
427	/* 14 */ "(Not Used)",
428	/* 15 */ "VMXON executed in VMX root operation.",
429	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
430	/* 17 */ "VM entry with non-launched executing VMCS.",
431	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
432	/* 19 */ "VMCALL with non-clear VMCS.",
433	/* 20 */ "VMCALL with invalid VM-exit control fields.",
434	/* 21 */ "(Not Used)",
435	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
436	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
437	/* 24 */ "VMCALL with invalid SMM-monitor features.",
438	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
439	/* 26 */ "VM entry with events blocked by MOV SS.",
440	/* 27 */ "(Not Used)",
441	/* 28 */ "Invalid operand to INVEPT/INVVPID."
442	};
443	#endif /* VBOX_STRICT */
444
445
446
447	/**
448	* Updates the VM's last error record. If there was a VMX instruction error,
449	* reads the error data from the VMCS and updates VCPU's last error record as
450	* well.
451	*
452	* @param pVM Pointer to the VM.
453	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
454	* VERR_VMX_UNABLE_TO_START_VM or
455	* VERR_VMX_INVALID_VMCS_FIELD).
456	* @param rc The error code.
457	*/
458	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
459	{
460	AssertPtr(pVM);
461	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
462	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
463	{
464	AssertPtrReturnVoid(pVCpu);
465	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
466	}
467	pVM->hm.s.lLastError = rc;
468	}
469
470
471	/**
472	* Reads the VM-entry interruption-information field from the VMCS into the VMX
473	* transient structure.
474	*
475	* @returns VBox status code.
476	* @param pVmxTransient Pointer to the VMX transient structure.
477	*
478	* @remarks No-long-jump zone!!!
479	*/
480	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
481	{
482	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
483	AssertRCReturn(rc, rc);
484	return VINF_SUCCESS;
485	}
486
487
488	/**
489	* Reads the VM-entry exception error code field from the VMCS into
490	* the VMX transient structure.
491	*
492	* @returns VBox status code.
493	* @param pVmxTransient Pointer to the VMX transient structure.
494	*
495	* @remarks No-long-jump zone!!!
496	*/
497	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
498	{
499	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
500	AssertRCReturn(rc, rc);
501	return VINF_SUCCESS;
502	}
503
504
505	/**
506	* Reads the VM-entry exception error code field from the VMCS into
507	* the VMX transient structure.
508	*
509	* @returns VBox status code.
510	* @param pVCpu Pointer to the VMCPU.
511	* @param pVmxTransient Pointer to the VMX transient structure.
512	*
513	* @remarks No-long-jump zone!!!
514	*/
515	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
516	{
517	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
518	AssertRCReturn(rc, rc);
519	return VINF_SUCCESS;
520	}
521
522
523	/**
524	* Reads the VM-exit interruption-information field from the VMCS into the VMX
525	* transient structure.
526	*
527	* @returns VBox status code.
528	* @param pVCpu Pointer to the VMCPU.
529	* @param pVmxTransient Pointer to the VMX transient structure.
530	*/
531	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
532	{
533	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
534	{
535	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
536	AssertRCReturn(rc, rc);
537	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
538	}
539	return VINF_SUCCESS;
540	}
541
542
543	/**
544	* Reads the VM-exit interruption error code from the VMCS into the VMX
545	* transient structure.
546	*
547	* @returns VBox status code.
548	* @param pVCpu Pointer to the VMCPU.
549	* @param pVmxTransient Pointer to the VMX transient structure.
550	*/
551	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
552	{
553	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
554	{
555	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
556	AssertRCReturn(rc, rc);
557	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
558	}
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Reads the VM-exit instruction length field from the VMCS into the VMX
565	* transient structure.
566	*
567	* @returns VBox status code.
568	* @param pVCpu Pointer to the VMCPU.
569	* @param pVmxTransient Pointer to the VMX transient structure.
570	*/
571	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
572	{
573	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
574	{
575	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
576	AssertRCReturn(rc, rc);
577	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
578	}
579	return VINF_SUCCESS;
580	}
581
582
583	/**
584	* Reads the exit qualification from the VMCS into the VMX transient structure.
585	*
586	* @returns VBox status code.
587	* @param pVCpu Pointer to the VMCPU.
588	* @param pVmxTransient Pointer to the VMX transient structure.
589	*/
590	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
591	{
592	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
593	{
594	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
595	AssertRCReturn(rc, rc);
596	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
597	}
598	return VINF_SUCCESS;
599	}
600
601
602	/**
603	* Reads the IDT-vectoring information field from the VMCS into the VMX
604	* transient structure.
605	*
606	* @returns VBox status code.
607	* @param pVmxTransient Pointer to the VMX transient structure.
608	*
609	* @remarks No-long-jump zone!!!
610	*/
611	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
612	{
613	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
614	{
615	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
616	AssertRCReturn(rc, rc);
617	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
618	}
619	return VINF_SUCCESS;
620	}
621
622
623	/**
624	* Reads the IDT-vectoring error code from the VMCS into the VMX
625	* transient structure.
626	*
627	* @returns VBox status code.
628	* @param pVmxTransient Pointer to the VMX transient structure.
629	*/
630	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
631	{
632	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
633	{
634	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
635	AssertRCReturn(rc, rc);
636	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
637	}
638	return VINF_SUCCESS;
639	}
640
641
642	/**
643	* Enters VMX root mode operation on the current CPU.
644	*
645	* @returns VBox status code.
646	* @param pVM Pointer to the VM (optional, can be NULL, after
647	* a resume).
648	* @param HCPhysCpuPage Physical address of the VMXON region.
649	* @param pvCpuPage Pointer to the VMXON region.
650	*/
651	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
652	{
653	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
654	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
655	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
656
657	if (pVM)
658	{
659	/* Write the VMCS revision dword to the VMXON region. */
660	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
661	}
662
663	/* Enable the VMX bit in CR4 if necessary. */
664	RTCCUINTREG uCr4 = ASMGetCR4();
665	if (!(uCr4 & X86_CR4_VMXE))
666	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
667
668	/* Enter VMX root mode. */
669	int rc = VMXEnable(HCPhysCpuPage);
670	if (RT_FAILURE(rc))
671	ASMSetCR4(uCr4);
672
673	return rc;
674	}
675
676
677	/**
678	* Exits VMX root mode operation on the current CPU.
679	*
680	* @returns VBox status code.
681	*/
682	static int hmR0VmxLeaveRootMode(void)
683	{
684	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
685
686	/* If we're for some reason not in VMX root mode, then don't leave it. */
687	RTCCUINTREG uHostCR4 = ASMGetCR4();
688	if (uHostCR4 & X86_CR4_VMXE)
689	{
690	/* Exit VMX root mode and clear the VMX bit in CR4. */
691	VMXDisable();
692	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
693	return VINF_SUCCESS;
694	}
695
696	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
697	}
698
699
700	/**
701	* Allocates and maps one physically contiguous page. The allocated page is
702	* zero'd out. (Used by various VT-x structures).
703	*
704	* @returns IPRT status code.
705	* @param pMemObj Pointer to the ring-0 memory object.
706	* @param ppVirt Where to store the virtual address of the
707	* allocation.
708	* @param pPhys Where to store the physical address of the
709	* allocation.
710	*/
711	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
712	{
713	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
714	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
715	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
716
717	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
718	if (RT_FAILURE(rc))
719	return rc;
720	ppVirt = RTR0MemObjAddress(pMemObj);
721	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
722	ASMMemZero32(*ppVirt, PAGE_SIZE);
723	return VINF_SUCCESS;
724	}
725
726
727	/**
728	* Frees and unmaps an allocated physical page.
729	*
730	* @param pMemObj Pointer to the ring-0 memory object.
731	* @param ppVirt Where to re-initialize the virtual address of
732	* allocation as 0.
733	* @param pHCPhys Where to re-initialize the physical address of the
734	* allocation as 0.
735	*/
736	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
737	{
738	AssertPtr(pMemObj);
739	AssertPtr(ppVirt);
740	AssertPtr(pHCPhys);
741	if (*pMemObj != NIL_RTR0MEMOBJ)
742	{
743	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
744	AssertRC(rc);
745	*pMemObj = NIL_RTR0MEMOBJ;
746	*ppVirt = 0;
747	*pHCPhys = 0;
748	}
749	}
750
751
752	/**
753	* Worker function to free VT-x related structures.
754	*
755	* @returns IPRT status code.
756	* @param pVM Pointer to the VM.
757	*/
758	static void hmR0VmxStructsFree(PVM pVM)
759	{
760	for (VMCPUID i = 0; i < pVM->cCpus; i++)
761	{
762	PVMCPU pVCpu = &pVM->aCpus[i];
763	AssertPtr(pVCpu);
764
765	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
766	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
767	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
768	#endif
769
770	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
771	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
772
773	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
774	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
775	}
776
777	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
778	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
779	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
780	#endif
781	}
782
783
784	/**
785	* Worker function to allocate VT-x related VM structures.
786	*
787	* @returns IPRT status code.
788	* @param pVM Pointer to the VM.
789	*/
790	static int hmR0VmxStructsAlloc(PVM pVM)
791	{
792	/*
793	* Initialize members up-front so we can cleanup properly on allocation failure.
794	*/
795	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
796	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
797	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
798	pVM->hm.s.vmx.HCPhys##a_Name = 0;
799
800	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
801	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
802	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
803	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
804
805	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
806	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
807	#endif
808	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
809
810	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
811	for (VMCPUID i = 0; i < pVM->cCpus; i++)
812	{
813	PVMCPU pVCpu = &pVM->aCpus[i];
814	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
815	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
816	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
817	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
818	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
819	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
820	#endif
821	}
822	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
823	#undef VMXLOCAL_INIT_VM_MEMOBJ
824
825	/*
826	* Allocate all the VT-x structures.
827	*/
828	int rc = VINF_SUCCESS;
829	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
830	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
831	if (RT_FAILURE(rc))
832	goto cleanup;
833	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
834	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
835	#endif
836
837	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
838	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
839	{
840	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
841	&pVM->hm.s.vmx.HCPhysApicAccess);
842	if (RT_FAILURE(rc))
843	goto cleanup;
844	}
845
846	/*
847	* Initialize per-VCPU VT-x structures.
848	*/
849	for (VMCPUID i = 0; i < pVM->cCpus; i++)
850	{
851	PVMCPU pVCpu = &pVM->aCpus[i];
852	AssertPtr(pVCpu);
853
854	/* Allocate the VM control structure (VMCS). */
855	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
856	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
857	if (RT_FAILURE(rc))
858	goto cleanup;
859
860	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
861	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
862	{
863	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
864	&pVCpu->hm.s.vmx.HCPhysVirtApic);
865	if (RT_FAILURE(rc))
866	goto cleanup;
867	}
868
869	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
870	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
871	{
872	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
873	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
874	if (RT_FAILURE(rc))
875	goto cleanup;
876	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
877	}
878
879	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
880	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
881	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
882	if (RT_FAILURE(rc))
883	goto cleanup;
884
885	/* Allocate the VM-exit MSR-load page for the host MSRs. */
886	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
887	if (RT_FAILURE(rc))
888	goto cleanup;
889	#endif
890	}
891
892	return VINF_SUCCESS;
893
894	cleanup:
895	hmR0VmxStructsFree(pVM);
896	return rc;
897	}
898
899
900	/**
901	* Does global VT-x initialization (called during module initialization).
902	*
903	* @returns VBox status code.
904	*/
905	VMMR0DECL(int) VMXR0GlobalInit(void)
906	{
907	#ifdef HMVMX_USE_FUNCTION_TABLE
908	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
909	# ifdef VBOX_STRICT
910	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
911	Assert(g_apfnVMExitHandlers[i]);
912	# endif
913	#endif
914	return VINF_SUCCESS;
915	}
916
917
918	/**
919	* Does global VT-x termination (called during module termination).
920	*/
921	VMMR0DECL(void) VMXR0GlobalTerm()
922	{
923	/* Nothing to do currently. */
924	}
925
926
927	/**
928	* Sets up and activates VT-x on the current CPU.
929	*
930	* @returns VBox status code.
931	* @param pCpu Pointer to the global CPU info struct.
932	* @param pVM Pointer to the VM (can be NULL after a host resume
933	* operation).
934	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
935	* fEnabledByHost is true).
936	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
937	* @a fEnabledByHost is true).
938	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
939	* enable VT-x on the host.
940	*/
941	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
942	{
943	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
944	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
945
946	if (!fEnabledByHost)
947	{
948	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
949	if (RT_FAILURE(rc))
950	return rc;
951	}
952
953	/*
954	* Flush all EPTP tagged-TLB entries (in case any other hypervisor have been using EPTPs) so that
955	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
956	* each time while reusing a VPID after hitting the MaxASID limit once.
957	*/
958	if ( pVM
959	&& pVM->hm.s.fNestedPaging)
960	{
961	/* We require ALL_CONTEXT flush-type to be available on the CPU. See hmR0VmxSetupTaggedTlb(). */
962	Assert(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
963	hmR0VmxFlushEpt(pVM, NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
964	pCpu->fFlushAsidBeforeUse = false;
965	}
966	else
967	{
968	/** @todo This is still not perfect. If on host resume (pVM is NULL or a VM
969	* without Nested Paging triggered this function) we still have the risk
970	* of potentially running with stale TLB-entries from other hypervisors
971	* when later we use a VM with NestedPaging. To fix this properly we will
972	* have to pass '&g_HvmR0' (see HMR0.cpp) to this function and read
973	* 'vmx_ept_vpid_caps' from it. Sigh. */
974	pCpu->fFlushAsidBeforeUse = true;
975	}
976
977	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
978	++pCpu->cTlbFlushes;
979
980	return VINF_SUCCESS;
981	}
982
983
984	/**
985	* Deactivates VT-x on the current CPU.
986	*
987	* @returns VBox status code.
988	* @param pCpu Pointer to the global CPU info struct.
989	* @param pvCpuPage Pointer to the VMXON region.
990	* @param HCPhysCpuPage Physical address of the VMXON region.
991	*
992	* @remarks This function should never be called when SUPR0EnableVTx() or
993	* similar was used to enable VT-x on the host.
994	*/
995	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
996	{
997	NOREF(pCpu);
998	NOREF(pvCpuPage);
999	NOREF(HCPhysCpuPage);
1000
1001	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1002	return hmR0VmxLeaveRootMode();
1003	}
1004
1005
1006	/**
1007	* Sets the permission bits for the specified MSR in the MSR bitmap.
1008	*
1009	* @param pVCpu Pointer to the VMCPU.
1010	* @param uMSR The MSR value.
1011	* @param enmRead Whether reading this MSR causes a VM-exit.
1012	* @param enmWrite Whether writing this MSR causes a VM-exit.
1013	*/
1014	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1015	{
1016	int32_t iBit;
1017	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1018
1019	/*
1020	* Layout:
1021	* 0x000 - 0x3ff - Low MSR read bits
1022	* 0x400 - 0x7ff - High MSR read bits
1023	* 0x800 - 0xbff - Low MSR write bits
1024	* 0xc00 - 0xfff - High MSR write bits
1025	*/
1026	if (uMsr <= 0x00001FFF)
1027	iBit = uMsr;
1028	else if ( uMsr >= 0xC0000000
1029	&& uMsr <= 0xC0001FFF)
1030	{
1031	iBit = (uMsr - 0xC0000000);
1032	pbMsrBitmap += 0x400;
1033	}
1034	else
1035	{
1036	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1037	return;
1038	}
1039
1040	Assert(iBit <= 0x1fff);
1041	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1042	ASMBitSet(pbMsrBitmap, iBit);
1043	else
1044	ASMBitClear(pbMsrBitmap, iBit);
1045
1046	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1047	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1048	else
1049	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1050	}
1051
1052
1053	/**
1054	* Flushes the TLB using EPT.
1055	*
1056	* @returns VBox status code.
1057	* @param pVM Pointer to the VM.
1058	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1059	* enmFlush).
1060	* @param enmFlush Type of flush.
1061	*/
1062	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1063	{
1064	AssertPtr(pVM);
1065	Assert(pVM->hm.s.fNestedPaging);
1066
1067	uint64_t descriptor[2];
1068	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1069	descriptor[0] = 0;
1070	else
1071	{
1072	Assert(pVCpu);
1073	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1074	}
1075	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1076
1077	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1078	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1079	rc));
1080	if ( RT_SUCCESS(rc)
1081	&& pVCpu)
1082	{
1083	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1084	}
1085	}
1086
1087
1088	/**
1089	* Flushes the TLB using VPID.
1090	*
1091	* @returns VBox status code.
1092	* @param pVM Pointer to the VM.
1093	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1094	* enmFlush).
1095	* @param enmFlush Type of flush.
1096	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1097	* on @a enmFlush).
1098	*/
1099	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1100	{
1101	AssertPtr(pVM);
1102	Assert(pVM->hm.s.vmx.fVpid);
1103
1104	uint64_t descriptor[2];
1105	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1106	{
1107	descriptor[0] = 0;
1108	descriptor[1] = 0;
1109	}
1110	else
1111	{
1112	AssertPtr(pVCpu);
1113	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1114	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1115	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1116	descriptor[1] = GCPtr;
1117	}
1118
1119	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1120	AssertMsg(rc == VINF_SUCCESS,
1121	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1122	if ( RT_SUCCESS(rc)
1123	&& pVCpu)
1124	{
1125	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1126	}
1127	}
1128
1129
1130	/**
1131	* Invalidates a guest page by guest virtual address. Only relevant for
1132	* EPT/VPID, otherwise there is nothing really to invalidate.
1133	*
1134	* @returns VBox status code.
1135	* @param pVM Pointer to the VM.
1136	* @param pVCpu Pointer to the VMCPU.
1137	* @param GCVirt Guest virtual address of the page to invalidate.
1138	*/
1139	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1140	{
1141	AssertPtr(pVM);
1142	AssertPtr(pVCpu);
1143	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1144
1145	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1146	if (!fFlushPending)
1147	{
1148	/*
1149	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1150	* See @bugref{6043} and @bugref{6177}.
1151	*
1152	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1153	* function maybe called in a loop with individual addresses.
1154	*/
1155	if (pVM->hm.s.vmx.fVpid)
1156	{
1157	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1158	{
1159	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1160	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1161	}
1162	else
1163	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1164	}
1165	else if (pVM->hm.s.fNestedPaging)
1166	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1167	}
1168
1169	return VINF_SUCCESS;
1170	}
1171
1172
1173	/**
1174	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1175	* otherwise there is nothing really to invalidate.
1176	*
1177	* @returns VBox status code.
1178	* @param pVM Pointer to the VM.
1179	* @param pVCpu Pointer to the VMCPU.
1180	* @param GCPhys Guest physical address of the page to invalidate.
1181	*/
1182	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1183	{
1184	LogFlowFunc(("%RGp\n", GCPhys));
1185
1186	/*
1187	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1188	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1189	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1190	*/
1191	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1192	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1193	return VINF_SUCCESS;
1194	}
1195
1196
1197	/**
1198	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1199	* case where neither EPT nor VPID is supported by the CPU.
1200	*
1201	* @param pVM Pointer to the VM.
1202	* @param pVCpu Pointer to the VMCPU.
1203	*
1204	* @remarks Called with interrupts disabled.
1205	*/
1206	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1207	{
1208	NOREF(pVM);
1209	AssertPtr(pVCpu);
1210	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1211	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1212
1213	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1214	AssertPtr(pCpu);
1215
1216	pVCpu->hm.s.TlbShootdown.cPages = 0;
1217	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1218	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1219	pVCpu->hm.s.fForceTLBFlush = false;
1220	return;
1221	}
1222
1223
1224	/**
1225	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1226	*
1227	* @param pVM Pointer to the VM.
1228	* @param pVCpu Pointer to the VMCPU.
1229	* @remarks All references to "ASID" in this function pertains to "VPID" in
1230	* Intel's nomenclature. The reason is, to avoid confusion in compare
1231	* statements since the host-CPU copies are named "ASID".
1232	*
1233	* @remarks Called with interrupts disabled.
1234	*/
1235	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1236	{
1237	#ifdef VBOX_WITH_STATISTICS
1238	bool fTlbFlushed = false;
1239	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1240	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1241	if (!fTlbFlushed) \
1242	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1243	} while (0)
1244	#else
1245	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1246	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1247	#endif
1248
1249	AssertPtr(pVM);
1250	AssertPtr(pVCpu);
1251	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1252	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1253	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1254
1255	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1256	AssertPtr(pCpu);
1257
1258	/*
1259	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1260	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1261	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1262	*/
1263	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1264	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1265	{
1266	++pCpu->uCurrentAsid;
1267	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1268	{
1269	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1270	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1271	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1272	}
1273
1274	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1275	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1276	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1277
1278	/*
1279	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1280	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1281	*/
1282	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1283	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1284	HMVMX_SET_TAGGED_TLB_FLUSHED();
1285	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1286	}
1287
1288	/* Check for explicit TLB shootdowns. */
1289	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1290	{
1291	/*
1292	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1293	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1294	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1295	* but not guest-physical mappings.
1296	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1297	*/
1298	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1299	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1300	HMVMX_SET_TAGGED_TLB_FLUSHED();
1301	}
1302
1303	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1304	* not be executed. See hmQueueInvlPage() where it is commented
1305	* out. Support individual entry flushing someday. */
1306	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1307	{
1308	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1309
1310	/*
1311	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1312	* as supported by the CPU.
1313	*/
1314	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1315	{
1316	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1317	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1318	}
1319	else
1320	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1321
1322	HMVMX_SET_TAGGED_TLB_FLUSHED();
1323	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1324	}
1325
1326	pVCpu->hm.s.TlbShootdown.cPages = 0;
1327	pVCpu->hm.s.fForceTLBFlush = false;
1328
1329	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1330
1331	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1332	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1333	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1334	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1335	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1336	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1337	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1338	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1339
1340	/* Update VMCS with the VPID. */
1341	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1342	AssertRC(rc);
1343
1344	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1345	}
1346
1347
1348	/**
1349	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1350	*
1351	* @returns VBox status code.
1352	* @param pVM Pointer to the VM.
1353	* @param pVCpu Pointer to the VMCPU.
1354	*
1355	* @remarks Called with interrupts disabled.
1356	*/
1357	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1358	{
1359	AssertPtr(pVM);
1360	AssertPtr(pVCpu);
1361	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1362	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1363
1364	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1365	AssertPtr(pCpu);
1366
1367	/*
1368	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1369	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1370	*/
1371	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1372	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1373	{
1374	pVCpu->hm.s.fForceTLBFlush = true;
1375	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1376	}
1377
1378	/* Check for explicit TLB shootdown flushes. */
1379	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1380	{
1381	pVCpu->hm.s.fForceTLBFlush = true;
1382	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1383	}
1384
1385	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1386	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1387
1388	if (pVCpu->hm.s.fForceTLBFlush)
1389	{
1390	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1391	pVCpu->hm.s.fForceTLBFlush = false;
1392	}
1393	else
1394	{
1395	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1396	* not be executed. See hmQueueInvlPage() where it is commented
1397	* out. Support individual entry flushing someday. */
1398	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1399	{
1400	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1401	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1402	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1403	}
1404	else
1405	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1406	}
1407
1408	pVCpu->hm.s.TlbShootdown.cPages = 0;
1409	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1410	}
1411
1412
1413	/**
1414	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1415	*
1416	* @returns VBox status code.
1417	* @param pVM Pointer to the VM.
1418	* @param pVCpu Pointer to the VMCPU.
1419	*
1420	* @remarks Called with interrupts disabled.
1421	*/
1422	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1423	{
1424	AssertPtr(pVM);
1425	AssertPtr(pVCpu);
1426	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1427	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1428
1429	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1430
1431	/*
1432	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1433	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1434	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1435	*/
1436	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1437	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1438	{
1439	pVCpu->hm.s.fForceTLBFlush = true;
1440	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1441	}
1442
1443	/* Check for explicit TLB shootdown flushes. */
1444	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1445	{
1446	/*
1447	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1448	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1449	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1450	*/
1451	pVCpu->hm.s.fForceTLBFlush = true;
1452	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1453	}
1454
1455	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1456	if (pVCpu->hm.s.fForceTLBFlush)
1457	{
1458	++pCpu->uCurrentAsid;
1459	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1460	{
1461	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1462	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1463	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1464	}
1465
1466	pVCpu->hm.s.fForceTLBFlush = false;
1467	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1468	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1469	if (pCpu->fFlushAsidBeforeUse)
1470	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1471	}
1472	else
1473	{
1474	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1475	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1476	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1477	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1478
1479	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1480	* not be executed. See hmQueueInvlPage() where it is commented
1481	* out. Support individual entry flushing someday. */
1482	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1483	{
1484	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1485	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1486	{
1487	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1488	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1489	}
1490	else
1491	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1492	}
1493	else
1494	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1495	}
1496
1497	pVCpu->hm.s.TlbShootdown.cPages = 0;
1498	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1499
1500	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1501	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1502	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1503	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1504	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1505	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1506
1507	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1508	AssertRC(rc);
1509	}
1510
1511
1512	/**
1513	* Flushes the guest TLB entry based on CPU capabilities.
1514	*
1515	* @param pVCpu Pointer to the VMCPU.
1516	*/
1517	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1518	{
1519	PVM pVM = pVCpu->CTX_SUFF(pVM);
1520	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1521	{
1522	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1523	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1524	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1525	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1526	default:
1527	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1528	break;
1529	}
1530	}
1531
1532
1533	/**
1534	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1535	* TLB entries from the host TLB before VM-entry.
1536	*
1537	* @returns VBox status code.
1538	* @param pVM Pointer to the VM.
1539	*/
1540	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1541	{
1542	/*
1543	* Determine optimal flush type for Nested Paging.
1544	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1545	* guest execution (see hmR3InitFinalizeR0()).
1546	*/
1547	if (pVM->hm.s.fNestedPaging)
1548	{
1549	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1550	{
1551	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1552	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1553	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1554	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1555	else
1556	{
1557	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1558	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1559	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1560	}
1561
1562	/* Make sure the write-back cacheable memory type for EPT is supported. */
1563	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1564	{
1565	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1566	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1567	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1568	}
1569	}
1570	else
1571	{
1572	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1573	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1574	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1575	}
1576	}
1577
1578	/*
1579	* Determine optimal flush type for VPID.
1580	*/
1581	if (pVM->hm.s.vmx.fVpid)
1582	{
1583	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1584	{
1585	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1586	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1587	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1588	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1589	else
1590	{
1591	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1592	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1593	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1594	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1595	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1596	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1597	pVM->hm.s.vmx.fVpid = false;
1598	}
1599	}
1600	else
1601	{
1602	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1603	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1604	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1605	pVM->hm.s.vmx.fVpid = false;
1606	}
1607	}
1608
1609	/*
1610	* Setup the handler for flushing tagged-TLBs.
1611	*/
1612	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1613	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1614	else if (pVM->hm.s.fNestedPaging)
1615	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1616	else if (pVM->hm.s.vmx.fVpid)
1617	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1618	else
1619	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1620	return VINF_SUCCESS;
1621	}
1622
1623
1624	/**
1625	* Sets up pin-based VM-execution controls in the VMCS.
1626	*
1627	* @returns VBox status code.
1628	* @param pVM Pointer to the VM.
1629	* @param pVCpu Pointer to the VMCPU.
1630	*/
1631	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1632	{
1633	AssertPtr(pVM);
1634	AssertPtr(pVCpu);
1635
1636	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1637	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1638
1639	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1640	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1641	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1642
1643	/* Enable the VMX preemption timer. */
1644	if (pVM->hm.s.vmx.fUsePreemptTimer)
1645	{
1646	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1647	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1648	}
1649
1650	if ((val & zap) != val)
1651	{
1652	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1653	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1654	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1655	}
1656
1657	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1658	AssertRCReturn(rc, rc);
1659
1660	/* Update VCPU with the currently set pin-based VM-execution controls. */
1661	pVCpu->hm.s.vmx.u32PinCtls = val;
1662	return rc;
1663	}
1664
1665
1666	/**
1667	* Sets up processor-based VM-execution controls in the VMCS.
1668	*
1669	* @returns VBox status code.
1670	* @param pVM Pointer to the VM.
1671	* @param pVMCPU Pointer to the VMCPU.
1672	*/
1673	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1674	{
1675	AssertPtr(pVM);
1676	AssertPtr(pVCpu);
1677
1678	int rc = VERR_INTERNAL_ERROR_5;
1679	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1680	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1681
1682	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1683	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1684	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1685	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1686	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1687	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1688	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1689
1690	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1691	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1692	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1693	{
1694	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1695	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1696	}
1697
1698	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1699	if (!pVM->hm.s.fNestedPaging)
1700	{
1701	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1702	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1703	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1704	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1705	}
1706
1707	/* Use TPR shadowing if supported by the CPU. */
1708	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1709	{
1710	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1711	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1712	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1713	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1714	AssertRCReturn(rc, rc);
1715
1716	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1717	/* CR8 writes causes a VM-exit based on TPR threshold. */
1718	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1719	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1720	}
1721	else
1722	{
1723	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1724	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1725	}
1726
1727	/* Use MSR-bitmaps if supported by the CPU. */
1728	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1729	{
1730	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1731
1732	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1733	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1734	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1735	AssertRCReturn(rc, rc);
1736
1737	/*
1738	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1739	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1740	*/
1741	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1742	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1743	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1744	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1745	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1746	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1747	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1748	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1749	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1750	}
1751
1752	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1753	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1754	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1755
1756	if ((val & zap) != val)
1757	{
1758	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1759	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1760	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1761	}
1762
1763	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1764	AssertRCReturn(rc, rc);
1765
1766	/* Update VCPU with the currently set processor-based VM-execution controls. */
1767	pVCpu->hm.s.vmx.u32ProcCtls = val;
1768
1769	/*
1770	* Secondary processor-based VM-execution controls.
1771	*/
1772	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1773	{
1774	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1775	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1776
1777	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1778	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1779
1780	if (pVM->hm.s.fNestedPaging)
1781	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1782	else
1783	{
1784	/*
1785	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1786	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1787	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1788	*/
1789	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1790	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1791	}
1792
1793	if (pVM->hm.s.vmx.fVpid)
1794	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1795
1796	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1797	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1798
1799	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1800	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1801	* done dynamically. */
1802	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1803	{
1804	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1805	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1806	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1807	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1808	AssertRCReturn(rc, rc);
1809	}
1810
1811	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1812	{
1813	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1814	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1815	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1816	}
1817
1818	if ((val & zap) != val)
1819	{
1820	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1821	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1822	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1823	}
1824
1825	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1826	AssertRCReturn(rc, rc);
1827
1828	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1829	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1830	}
1831
1832	return VINF_SUCCESS;
1833	}
1834
1835
1836	/**
1837	* Sets up miscellaneous (everything other than Pin & Processor-based
1838	* VM-execution) control fields in the VMCS.
1839	*
1840	* @returns VBox status code.
1841	* @param pVM Pointer to the VM.
1842	* @param pVCpu Pointer to the VMCPU.
1843	*/
1844	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1845	{
1846	AssertPtr(pVM);
1847	AssertPtr(pVCpu);
1848
1849	int rc = VERR_GENERAL_FAILURE;
1850
1851	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1852	#if 0
1853	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1854	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1855	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1856
1857	/*
1858	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1859	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1860	* We thus use the exception bitmap to control it rather than use both.
1861	*/
1862	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1863	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1864
1865	/** @todo Explore possibility of using IO-bitmaps. */
1866	/* All IO & IOIO instructions cause VM-exits. */
1867	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1868	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1869
1870	/* Initialize the MSR-bitmap area. */
1871	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1872	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1873	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1874	#endif
1875
1876	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1877	/* Setup MSR autoloading/storing. */
1878	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1879	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1880	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1881	AssertRCReturn(rc, rc);
1882	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1883	AssertRCReturn(rc, rc);
1884
1885	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1886	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1887	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1888	AssertRCReturn(rc, rc);
1889	#endif
1890
1891	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1892	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1893	AssertRCReturn(rc, rc);
1894
1895	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1896	#if 0
1897	/* Setup debug controls */
1898	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1899	AssertRCReturn(rc, rc);
1900	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1901	AssertRCReturn(rc, rc);
1902	#endif
1903
1904	return rc;
1905	}
1906
1907
1908	/**
1909	* Sets up the initial exception bitmap in the VMCS based on static conditions
1910	* (i.e. conditions that cannot ever change at runtime).
1911	*
1912	* @returns VBox status code.
1913	* @param pVM Pointer to the VM.
1914	* @param pVCpu Pointer to the VMCPU.
1915	*/
1916	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1917	{
1918	AssertPtr(pVM);
1919	AssertPtr(pVCpu);
1920
1921	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1922
1923	uint32_t u32XcptBitmap = 0;
1924
1925	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1926	if (!pVM->hm.s.fNestedPaging)
1927	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1928
1929	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1930	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1931	AssertRCReturn(rc, rc);
1932	return rc;
1933	}
1934
1935
1936	/**
1937	* Sets up the initial guest-state mask. The guest-state mask is consulted
1938	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1939	* for the nested virtualization case (as it would cause a VM-exit).
1940	*
1941	* @param pVCpu Pointer to the VMCPU.
1942	*/
1943	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1944	{
1945	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1946	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1947	return VINF_SUCCESS;
1948	}
1949
1950
1951	/**
1952	* Does per-VM VT-x initialization.
1953	*
1954	* @returns VBox status code.
1955	* @param pVM Pointer to the VM.
1956	*/
1957	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1958	{
1959	LogFlowFunc(("pVM=%p\n", pVM));
1960
1961	int rc = hmR0VmxStructsAlloc(pVM);
1962	if (RT_FAILURE(rc))
1963	{
1964	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1965	return rc;
1966	}
1967
1968	return VINF_SUCCESS;
1969	}
1970
1971
1972	/**
1973	* Does per-VM VT-x termination.
1974	*
1975	* @returns VBox status code.
1976	* @param pVM Pointer to the VM.
1977	*/
1978	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1979	{
1980	LogFlowFunc(("pVM=%p\n", pVM));
1981
1982	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1983	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1984	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1985	#endif
1986	hmR0VmxStructsFree(pVM);
1987	return VINF_SUCCESS;
1988	}
1989
1990
1991	/**
1992	* Sets up the VM for execution under VT-x.
1993	* This function is only called once per-VM during initialization.
1994	*
1995	* @returns VBox status code.
1996	* @param pVM Pointer to the VM.
1997	*/
1998	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1999	{
2000	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2001	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2002
2003	LogFlowFunc(("pVM=%p\n", pVM));
2004
2005	/*
2006	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2007	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2008	*/
2009	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2010	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2011	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2012	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2013	{
2014	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2015	return VERR_INTERNAL_ERROR;
2016	}
2017
2018	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2019	/*
2020	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2021	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2022	*/
2023	if ( pVM->hm.s.fAllow64BitGuests
2024	&& !HMVMX_IS_64BIT_HOST_MODE())
2025	{
2026	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2027	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2028	}
2029	#endif
2030
2031	/* Initialize these always, see hmR3InitFinalizeR0().*/
2032	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2033	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2034
2035	/* Setup the tagged-TLB flush handlers. */
2036	int rc = hmR0VmxSetupTaggedTlb(pVM);
2037	if (RT_FAILURE(rc))
2038	{
2039	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2040	return rc;
2041	}
2042
2043	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2044	{
2045	PVMCPU pVCpu = &pVM->aCpus[i];
2046	AssertPtr(pVCpu);
2047	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2048
2049	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2050	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2051
2052	/* Set revision dword at the beginning of the VMCS structure. */
2053	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
2054
2055	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2056	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2057	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2058	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2059
2060	/* Load this VMCS as the current VMCS. */
2061	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2062	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2063	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2064
2065	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2066	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2067	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2068
2069	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2070	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2071	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2072
2073	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2074	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2075	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2076
2077	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2078	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2079	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2080
2081	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2082	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2083	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2084
2085	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2086	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2087	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2088	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2089	#endif
2090
2091	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2092	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2093	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2094	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2095
2096	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2097	}
2098
2099	return VINF_SUCCESS;
2100	}
2101
2102
2103	/**
2104	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2105	* the VMCS.
2106	*
2107	* @returns VBox status code.
2108	* @param pVM Pointer to the VM.
2109	* @param pVCpu Pointer to the VMCPU.
2110	*/
2111	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2112	{
2113	RTCCUINTREG uReg = ASMGetCR0();
2114	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2115	AssertRCReturn(rc, rc);
2116
2117	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2118	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2119	if (HMVMX_IS_64BIT_HOST_MODE())
2120	{
2121	uint64_t uRegCR3 = HMR0Get64bitCR3();
2122	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2123	}
2124	else
2125	#endif
2126	{
2127	uReg = ASMGetCR3();
2128	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2129	}
2130	AssertRCReturn(rc, rc);
2131
2132	uReg = ASMGetCR4();
2133	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2134	AssertRCReturn(rc, rc);
2135	return rc;
2136	}
2137
2138
2139	/**
2140	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2141	* the host-state area in the VMCS.
2142	*
2143	* @returns VBox status code.
2144	* @param pVM Pointer to the VM.
2145	* @param pVCpu Pointer to the VMCPU.
2146	*/
2147	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2148	{
2149	int rc = VERR_INTERNAL_ERROR_5;
2150	RTSEL uSelDS = 0;
2151	RTSEL uSelES = 0;
2152	RTSEL uSelFS = 0;
2153	RTSEL uSelGS = 0;
2154	RTSEL uSelTR = 0;
2155
2156	/*
2157	* Host DS, ES, FS and GS segment registers.
2158	*/
2159	#if HC_ARCH_BITS == 64
2160	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2161	uSelDS = ASMGetDS();
2162	uSelES = ASMGetES();
2163	uSelFS = ASMGetFS();
2164	uSelGS = ASMGetGS();
2165	#endif
2166
2167	/*
2168	* Host CS and SS segment registers.
2169	*/
2170	RTSEL uSelCS;
2171	RTSEL uSelSS;
2172	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2173	if (HMVMX_IS_64BIT_HOST_MODE())
2174	{
2175	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2176	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2177	}
2178	else
2179	{
2180	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2181	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2182	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2183	}
2184	#else
2185	uSelCS = ASMGetCS();
2186	uSelSS = ASMGetSS();
2187	#endif
2188
2189	/*
2190	* Host TR segment register.
2191	*/
2192	uSelTR = ASMGetTR();
2193
2194	#if HC_ARCH_BITS == 64
2195	/*
2196	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2197	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2198	*/
2199	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2200	{
2201	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2202	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2203	uSelDS = 0;
2204	}
2205	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2206	{
2207	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2208	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2209	uSelES = 0;
2210	}
2211	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2212	{
2213	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2214	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2215	uSelFS = 0;
2216	}
2217	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2218	{
2219	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2220	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2221	uSelGS = 0;
2222	}
2223	#endif
2224
2225	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2226	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2227	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2228	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2229	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2230	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2231	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2232	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2233	Assert(uSelCS);
2234	Assert(uSelTR);
2235
2236	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2237	#if 0
2238	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2239	Assert(uSelSS != 0);
2240	#endif
2241
2242	/* Write these host selector fields into the host-state area in the VMCS. */
2243	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2244	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2245	#if HC_ARCH_BITS == 64
2246	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2247	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2248	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2249	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2250	#endif
2251	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2252
2253	/*
2254	* Host GDTR and IDTR.
2255	*/
2256	RTGDTR Gdtr;
2257	RT_ZERO(Gdtr);
2258	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2259	if (HMVMX_IS_64BIT_HOST_MODE())
2260	{
2261	X86XDTR64 Gdtr64;
2262	X86XDTR64 Idtr64;
2263	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2264	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2265	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2266
2267	Gdtr.cbGdt = Gdtr64.cb;
2268	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2269	}
2270	else
2271	#endif
2272	{
2273	RTIDTR Idtr;
2274	ASMGetGDTR(&Gdtr);
2275	ASMGetIDTR(&Idtr);
2276	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2277	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2278
2279	#if HC_ARCH_BITS == 64
2280	/*
2281	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2282	* maximum limit (0xffff) on every VM-exit.
2283	*/
2284	if (Gdtr.cbGdt != 0xffff)
2285	{
2286	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2287	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2288	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2289	}
2290
2291	/*
2292	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2293	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2294	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2295	*/
2296	if (Idtr.cbIdt < 0x0fff)
2297	{
2298	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2299	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2300	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2301	}
2302	#endif
2303	}
2304
2305	/*
2306	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2307	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2308	*/
2309	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2310	{
2311	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2312	return VERR_VMX_INVALID_HOST_STATE;
2313	}
2314
2315	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2316	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2317	if (HMVMX_IS_64BIT_HOST_MODE())
2318	{
2319	/* We need the 64-bit TR base for hybrid darwin. */
2320	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2321	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2322	}
2323	else
2324	#endif
2325	{
2326	uintptr_t uTRBase;
2327	#if HC_ARCH_BITS == 64
2328	uTRBase = X86DESC64_BASE(pDesc);
2329
2330	/*
2331	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2332	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2333	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2334	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2335	*
2336	* [1] See Intel spec. 3.5 "System Descriptor Types".
2337	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2338	*/
2339	Assert(pDesc->System.u4Type == 11);
2340	if ( pDesc->System.u16LimitLow != 0x67
2341	\|\| pDesc->System.u4LimitHigh)
2342	{
2343	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2344	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2345
2346	/* Store the GDTR here as we need it while restoring TR. */
2347	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2348	}
2349	#else
2350	uTRBase = X86DESC_BASE(pDesc);
2351	#endif
2352	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2353	}
2354	AssertRCReturn(rc, rc);
2355
2356	/*
2357	* Host FS base and GS base.
2358	*/
2359	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2360	if (HMVMX_IS_64BIT_HOST_MODE())
2361	{
2362	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2363	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2364	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2365	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2366
2367	# if HC_ARCH_BITS == 64
2368	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2369	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2370	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2371	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2372	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2373	# endif
2374	}
2375	#endif
2376	return rc;
2377	}
2378
2379
2380	/**
2381	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2382	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2383	* the host after every successful VM exit.
2384	*
2385	* @returns VBox status code.
2386	* @param pVM Pointer to the VM.
2387	* @param pVCpu Pointer to the VMCPU.
2388	*/
2389	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2390	{
2391	AssertPtr(pVCpu);
2392	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2393
2394	int rc = VINF_SUCCESS;
2395	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2396	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2397	uint32_t cHostMsrs = 0;
2398	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2399
2400	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2401	{
2402	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2403
2404	# if HC_ARCH_BITS == 64
2405	/* Paranoia. 64-bit code requires these bits to be set always. */
2406	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2407
2408	/*
2409	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2410	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2411	* some reason (e.g. allow transparent reads) we would activate the code below.
2412	*/
2413	# if 0
2414	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2415	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2416	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2417	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2418	if (CPUMIsGuestInLongMode(pVCpu))
2419	{
2420	uint64_t u64GuestEfer;
2421	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2422	AssertRC(rc);
2423
2424	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2425	{
2426	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2427	pHostMsr->u32Reserved = 0;
2428	pHostMsr->u64Value = u64HostEfer;
2429	pHostMsr++; cHostMsrs++;
2430	}
2431	}
2432	# endif
2433	# else /* HC_ARCH_BITS != 64 */
2434	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2435	pHostMsr->u32Reserved = 0;
2436	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2437	if (CPUMIsGuestInLongMode(pVCpu))
2438	{
2439	/* Must match the EFER value in our 64 bits switcher. */
2440	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2441	}
2442	else
2443	# endif
2444	pHostMsr->u64Value = u64HostEfer;
2445	pHostMsr++; cHostMsrs++;
2446	# endif /* HC_ARCH_BITS == 64 */
2447	}
2448
2449	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2450	if (HMVMX_IS_64BIT_HOST_MODE())
2451	{
2452	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2453	pHostMsr->u32Reserved = 0;
2454	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2455	pHostMsr++; cHostMsrs++;
2456	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2457	pHostMsr->u32Reserved = 0;
2458	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2459	pHostMsr++; cHostMsrs++;
2460	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2461	pHostMsr->u32Reserved = 0;
2462	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2463	pHostMsr++; cHostMsrs++;
2464	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2465	pHostMsr->u32Reserved = 0;
2466	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2467	pHostMsr++; cHostMsrs++;
2468	}
2469	# endif
2470
2471	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2472	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2473	{
2474	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2475	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2476	}
2477
2478	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2479	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2480
2481	/*
2482	* Host Sysenter MSRs.
2483	*/
2484	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2485	AssertRCReturn(rc, rc);
2486	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2487	if (HMVMX_IS_64BIT_HOST_MODE())
2488	{
2489	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2490	AssertRCReturn(rc, rc);
2491	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2492	}
2493	else
2494	{
2495	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2496	AssertRCReturn(rc, rc);
2497	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2498	}
2499	#elif HC_ARCH_BITS == 32
2500	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2501	AssertRCReturn(rc, rc);
2502	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2503	#else
2504	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2505	AssertRCReturn(rc, rc);
2506	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2507	#endif
2508	AssertRCReturn(rc, rc);
2509
2510	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2511	* hmR0VmxSetupExitCtls() !! */
2512	return rc;
2513	}
2514
2515
2516	/**
2517	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2518	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2519	* controls".
2520	*
2521	* @returns VBox status code.
2522	* @param pVCpu Pointer to the VMCPU.
2523	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2524	* out-of-sync. Make sure to update the required fields
2525	* before using them.
2526	*
2527	* @remarks No-long-jump zone!!!
2528	*/
2529	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2530	{
2531	int rc = VINF_SUCCESS;
2532	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2533	{
2534	PVM pVM = pVCpu->CTX_SUFF(pVM);
2535	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2536	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2537
2538	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2539	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2540
2541	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2542	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2543	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2544	else
2545	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2546
2547	/*
2548	* The following should not be set (since we're not in SMM mode):
2549	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2550	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2551	*/
2552
2553	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2554	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2555	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2556
2557	if ((val & zap) != val)
2558	{
2559	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2560	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2561	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2562	}
2563
2564	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2565	AssertRCReturn(rc, rc);
2566
2567	/* Update VCPU with the currently set VM-exit controls. */
2568	pVCpu->hm.s.vmx.u32EntryCtls = val;
2569	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2570	}
2571	return rc;
2572	}
2573
2574
2575	/**
2576	* Sets up the VM-exit controls in the VMCS.
2577	*
2578	* @returns VBox status code.
2579	* @param pVM Pointer to the VM.
2580	* @param pVCpu Pointer to the VMCPU.
2581	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2582	* out-of-sync. Make sure to update the required fields
2583	* before using them.
2584	*
2585	* @remarks requires EFER.
2586	*/
2587	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2588	{
2589	int rc = VINF_SUCCESS;
2590	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2591	{
2592	PVM pVM = pVCpu->CTX_SUFF(pVM);
2593	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2594	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2595
2596	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2597	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2598
2599	/*
2600	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2601	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2602	*/
2603	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2604	if (HMVMX_IS_64BIT_HOST_MODE())
2605	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2606	else
2607	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2608	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2609	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2610	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2611	else
2612	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2613	#endif
2614
2615	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2616	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2617
2618	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2619	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2620	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2621	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2622	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2623
2624	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2625	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2626
2627	if ((val & zap) != val)
2628	{
2629	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2630	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2631	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2632	}
2633
2634	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2635	AssertRCReturn(rc, rc);
2636
2637	/* Update VCPU with the currently set VM-exit controls. */
2638	pVCpu->hm.s.vmx.u32ExitCtls = val;
2639	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2640	}
2641	return rc;
2642	}
2643
2644
2645	/**
2646	* Loads the guest APIC and related state.
2647	*
2648	* @returns VBox status code.
2649	* @param pVM Pointer to the VM.
2650	* @param pVCpu Pointer to the VMCPU.
2651	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2652	* out-of-sync. Make sure to update the required fields
2653	* before using them.
2654	*/
2655	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2656	{
2657	int rc = VINF_SUCCESS;
2658	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2659	{
2660	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2661	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2662	{
2663	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2664
2665	bool fPendingIntr = false;
2666	uint8_t u8Tpr = 0;
2667	uint8_t u8PendingIntr = 0;
2668	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2669	AssertRCReturn(rc, rc);
2670
2671	/*
2672	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2673	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2674	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2675	* the interrupt when we VM-exit for other reasons.
2676	*/
2677	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2678	uint32_t u32TprThreshold = 0;
2679	if (fPendingIntr)
2680	{
2681	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2682	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2683	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2684	if (u8PendingPriority <= u8TprPriority)
2685	u32TprThreshold = u8PendingPriority;
2686	else
2687	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2688	}
2689	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2690
2691	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2692	AssertRCReturn(rc, rc);
2693	}
2694
2695	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2696	}
2697	return rc;
2698	}
2699
2700
2701	/**
2702	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2703	*
2704	* @returns Guest's interruptibility-state.
2705	* @param pVCpu Pointer to the VMCPU.
2706	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2707	* out-of-sync. Make sure to update the required fields
2708	* before using them.
2709	*
2710	* @remarks No-long-jump zone!!!
2711	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2712	*/
2713	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2714	{
2715	/*
2716	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2717	* inhibit interrupts or clear any existing interrupt-inhibition.
2718	*/
2719	uint32_t uIntrState = 0;
2720	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2721	{
2722	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2723	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2724	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2725	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2726	{
2727	/*
2728	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2729	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2730	*/
2731	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2732	}
2733	else if (pMixedCtx->eflags.Bits.u1IF)
2734	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2735	else
2736	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2737	}
2738	return uIntrState;
2739	}
2740
2741
2742	/**
2743	* Loads the guest's interruptibility-state into the guest-state area in the
2744	* VMCS.
2745	*
2746	* @returns VBox status code.
2747	* @param pVCpu Pointer to the VMCPU.
2748	* @param uIntrState The interruptibility-state to set.
2749	*/
2750	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2751	{
2752	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2753	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2754	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2755	AssertRCReturn(rc, rc);
2756	return rc;
2757	}
2758
2759
2760	/**
2761	* Loads the guest's RIP into the guest-state area in the VMCS.
2762	*
2763	* @returns VBox status code.
2764	* @param pVCpu Pointer to the VMCPU.
2765	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2766	* out-of-sync. Make sure to update the required fields
2767	* before using them.
2768	*
2769	* @remarks No-long-jump zone!!!
2770	*/
2771	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2772	{
2773	int rc = VINF_SUCCESS;
2774	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2775	{
2776	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2777	AssertRCReturn(rc, rc);
2778	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2779	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2780	}
2781	return rc;
2782	}
2783
2784
2785	/**
2786	* Loads the guest's RSP into the guest-state area in the VMCS.
2787	*
2788	* @returns VBox status code.
2789	* @param pVCpu Pointer to the VMCPU.
2790	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2791	* out-of-sync. Make sure to update the required fields
2792	* before using them.
2793	*
2794	* @remarks No-long-jump zone!!!
2795	*/
2796	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2797	{
2798	int rc = VINF_SUCCESS;
2799	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2800	{
2801	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2802	AssertRCReturn(rc, rc);
2803	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2804	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2805	}
2806	return rc;
2807	}
2808
2809
2810	/**
2811	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2812	*
2813	* @returns VBox status code.
2814	* @param pVCpu Pointer to the VMCPU.
2815	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2816	* out-of-sync. Make sure to update the required fields
2817	* before using them.
2818	*
2819	* @remarks No-long-jump zone!!!
2820	*/
2821	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2822	{
2823	int rc = VINF_SUCCESS;
2824	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2825	{
2826	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2827	Let us assert it as such and use 32-bit VMWRITE. */
2828	Assert(!(pMixedCtx->rflags.u64 >> 32));
2829	X86EFLAGS uEFlags = pMixedCtx->eflags;
2830	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2831	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2832
2833	/*
2834	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2835	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2836	*/
2837	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2838	{
2839	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2840	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2841	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2842	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2843	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2844	}
2845
2846	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2847	AssertRCReturn(rc, rc);
2848
2849	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2850	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2851	}
2852	return rc;
2853	}
2854
2855
2856	/**
2857	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2858	*
2859	* @returns VBox status code.
2860	* @param pVCpu Pointer to the VMCPU.
2861	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2862	* out-of-sync. Make sure to update the required fields
2863	* before using them.
2864	*
2865	* @remarks No-long-jump zone!!!
2866	*/
2867	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2868	{
2869	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2870	AssertRCReturn(rc, rc);
2871	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2872	AssertRCReturn(rc, rc);
2873	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2874	AssertRCReturn(rc, rc);
2875	return rc;
2876	}
2877
2878
2879	/**
2880	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2881	* in the VMCS.
2882	*
2883	* @returns VBox status code.
2884	* @param pVM Pointer to the VM.
2885	* @param pVCpu Pointer to the VMCPU.
2886	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2887	* out-of-sync. Make sure to update the required fields
2888	* before using them.
2889	*
2890	* @remarks No-long-jump zone!!!
2891	*/
2892	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2893	{
2894	int rc = VINF_SUCCESS;
2895	PVM pVM = pVCpu->CTX_SUFF(pVM);
2896
2897	/*
2898	* Guest CR0.
2899	* Guest FPU.
2900	*/
2901	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2902	{
2903	Assert(!(pCtx->cr0 >> 32));
2904	uint32_t u32GuestCR0 = pCtx->cr0;
2905
2906	/* The guest's view (read access) of its CR0 is unblemished. */
2907	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2908	AssertRCReturn(rc, rc);
2909	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2910
2911	/* Setup VT-x's view of the guest CR0. */
2912	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2913	if (pVM->hm.s.fNestedPaging)
2914	{
2915	if (CPUMIsGuestPagingEnabledEx(pCtx))
2916	{
2917	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2918	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2919	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2920	}
2921	else
2922	{
2923	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2924	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2925	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2926	}
2927
2928	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2929	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2930	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2931
2932	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2933	AssertRCReturn(rc, rc);
2934	}
2935	else
2936	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2937
2938	/*
2939	* Guest FPU bits.
2940	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2941	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2942	*/
2943	u32GuestCR0 \|= X86_CR0_NE;
2944	bool fInterceptNM = false;
2945	if (CPUMIsGuestFPUStateActive(pVCpu))
2946	{
2947	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2948	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2949	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2950	}
2951	else
2952	{
2953	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2954	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2955	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2956	}
2957
2958	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2959	bool fInterceptMF = false;
2960	if (!(pCtx->cr0 & X86_CR0_NE))
2961	fInterceptMF = true;
2962
2963	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2964	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2965	{
2966	Assert(PDMVmmDevHeapIsEnabled(pVM));
2967	Assert(pVM->hm.s.vmx.pRealModeTSS);
2968	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2969	fInterceptNM = true;
2970	fInterceptMF = true;
2971	}
2972	else
2973	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2974
2975	if (fInterceptNM)
2976	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2977	else
2978	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2979
2980	if (fInterceptMF)
2981	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2982	else
2983	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2984
2985	/* Additional intercepts for debugging, define these yourself explicitly. */
2986	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2987	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
2988	\| RT_BIT(X86_XCPT_BP)
2989	\| RT_BIT(X86_XCPT_DB)
2990	\| RT_BIT(X86_XCPT_DE)
2991	\| RT_BIT(X86_XCPT_NM)
2992	\| RT_BIT(X86_XCPT_UD)
2993	\| RT_BIT(X86_XCPT_NP)
2994	\| RT_BIT(X86_XCPT_SS)
2995	\| RT_BIT(X86_XCPT_GP)
2996	\| RT_BIT(X86_XCPT_PF)
2997	\| RT_BIT(X86_XCPT_MF)
2998	;
2999	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3000	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3001	#endif
3002
3003	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3004
3005	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3006	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3007	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3008	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3009	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3010	else
3011	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3012
3013	u32GuestCR0 \|= uSetCR0;
3014	u32GuestCR0 &= uZapCR0;
3015	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3016
3017	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3018	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3019	AssertRCReturn(rc, rc);
3020	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3021	AssertRCReturn(rc, rc);
3022	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3023
3024	/*
3025	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3026	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3027	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3028	*/
3029	uint32_t u32CR0Mask = 0;
3030	u32CR0Mask = X86_CR0_PE
3031	\| X86_CR0_NE
3032	\| X86_CR0_WP
3033	\| X86_CR0_PG
3034	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3035	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3036	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3037	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3038	u32CR0Mask &= ~X86_CR0_PE;
3039	if (pVM->hm.s.fNestedPaging)
3040	u32CR0Mask &= ~X86_CR0_WP;
3041
3042	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3043	if (fInterceptNM)
3044	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
3045	else
3046	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
3047
3048	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3049	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3050	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3051	AssertRCReturn(rc, rc);
3052
3053	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3054	}
3055
3056	/*
3057	* Guest CR2.
3058	* It's always loaded in the assembler code. Nothing to do here.
3059	*/
3060
3061	/*
3062	* Guest CR3.
3063	*/
3064	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3065	{
3066	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3067	if (pVM->hm.s.fNestedPaging)
3068	{
3069	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3070
3071	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3072	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3073	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3074	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3075
3076	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3077	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3078	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3079
3080	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3081	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3082	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3083	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3084
3085	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3086	AssertRCReturn(rc, rc);
3087	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3088
3089	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3090	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3091	{
3092	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3093	if (CPUMIsGuestInPAEModeEx(pCtx))
3094	{
3095	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3096	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3097	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3098	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3099	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3100	}
3101
3102	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3103	have Unrestricted Execution to handle the guest when it's not using paging. */
3104	GCPhysGuestCR3 = pCtx->cr3;
3105	}
3106	else
3107	{
3108	/*
3109	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3110	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3111	* EPT takes care of translating it to host-physical addresses.
3112	*/
3113	RTGCPHYS GCPhys;
3114	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3115	Assert(PDMVmmDevHeapIsEnabled(pVM));
3116
3117	/* We obtain it here every time as the guest could have relocated this PCI region. */
3118	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3119	AssertRCReturn(rc, rc);
3120
3121	GCPhysGuestCR3 = GCPhys;
3122	}
3123
3124	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3125	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3126	}
3127	else
3128	{
3129	/* Non-nested paging case, just use the hypervisor's CR3. */
3130	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3131
3132	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3133	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3134	}
3135	AssertRCReturn(rc, rc);
3136
3137	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3138	}
3139
3140	/*
3141	* Guest CR4.
3142	*/
3143	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3144	{
3145	Assert(!(pCtx->cr4 >> 32));
3146	uint32_t u32GuestCR4 = pCtx->cr4;
3147
3148	/* The guest's view of its CR4 is unblemished. */
3149	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3150	AssertRCReturn(rc, rc);
3151	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3152
3153	/* Setup VT-x's view of the guest CR4. */
3154	/*
3155	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3156	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3157	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3158	*/
3159	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3160	{
3161	Assert(pVM->hm.s.vmx.pRealModeTSS);
3162	Assert(PDMVmmDevHeapIsEnabled(pVM));
3163	u32GuestCR4 &= ~X86_CR4_VME;
3164	}
3165
3166	if (pVM->hm.s.fNestedPaging)
3167	{
3168	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3169	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3170	{
3171	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3172	u32GuestCR4 \|= X86_CR4_PSE;
3173	/* Our identity mapping is a 32 bits page directory. */
3174	u32GuestCR4 &= ~X86_CR4_PAE;
3175	}
3176	/* else use guest CR4.*/
3177	}
3178	else
3179	{
3180	/*
3181	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3182	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3183	*/
3184	switch (pVCpu->hm.s.enmShadowMode)
3185	{
3186	case PGMMODE_REAL: /* Real-mode. */
3187	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3188	case PGMMODE_32_BIT: /* 32-bit paging. */
3189	{
3190	u32GuestCR4 &= ~X86_CR4_PAE;
3191	break;
3192	}
3193
3194	case PGMMODE_PAE: /* PAE paging. */
3195	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3196	{
3197	u32GuestCR4 \|= X86_CR4_PAE;
3198	break;
3199	}
3200
3201	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3202	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3203	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3204	break;
3205	#endif
3206	default:
3207	AssertFailed();
3208	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3209	}
3210	}
3211
3212	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3213	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3214	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3215	u32GuestCR4 \|= uSetCR4;
3216	u32GuestCR4 &= uZapCR4;
3217
3218	/* Write VT-x's view of the guest CR4 into the VMCS. */
3219	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3220	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3221	AssertRCReturn(rc, rc);
3222
3223	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3224	uint32_t u32CR4Mask = 0;
3225	u32CR4Mask = X86_CR4_VME
3226	\| X86_CR4_PAE
3227	\| X86_CR4_PGE
3228	\| X86_CR4_PSE
3229	\| X86_CR4_VMXE;
3230	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3231	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3232	AssertRCReturn(rc, rc);
3233
3234	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3235	}
3236	return rc;
3237	}
3238
3239
3240	/**
3241	* Loads the guest debug registers into the guest-state area in the VMCS.
3242	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3243	*
3244	* @returns VBox status code.
3245	* @param pVCpu Pointer to the VMCPU.
3246	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3247	* out-of-sync. Make sure to update the required fields
3248	* before using them.
3249	*
3250	* @remarks No-long-jump zone!!!
3251	*/
3252	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3253	{
3254	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3255	return VINF_SUCCESS;
3256
3257	#ifdef VBOX_STRICT
3258	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3259	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3260	{
3261	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3262	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3263	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3264	}
3265	#endif
3266
3267	int rc = VERR_INTERNAL_ERROR_5;
3268	PVM pVM = pVCpu->CTX_SUFF(pVM);
3269	bool fInterceptDB = false;
3270	bool fInterceptMovDRx = false;
3271	if (DBGFIsStepping(pVCpu))
3272	{
3273	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3274	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3275	{
3276	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3277	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3278	AssertRCReturn(rc, rc);
3279	Assert(fInterceptDB == false);
3280	}
3281	else
3282	{
3283	fInterceptDB = true;
3284	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3285	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3286	}
3287	}
3288
3289	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3290	{
3291	if (!CPUMIsHyperDebugStateActive(pVCpu))
3292	{
3293	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3294	AssertRC(rc);
3295	}
3296	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3297	fInterceptMovDRx = true;
3298	}
3299	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3300	{
3301	if (!CPUMIsGuestDebugStateActive(pVCpu))
3302	{
3303	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3304	AssertRC(rc);
3305	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3306	}
3307	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3308	Assert(fInterceptMovDRx == false);
3309	}
3310	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3311	{
3312	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3313	fInterceptMovDRx = true;
3314	}
3315
3316	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3317	if (fInterceptDB)
3318	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3319	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3320	{
3321	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3322	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3323	#endif
3324	}
3325
3326	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3327	if (fInterceptMovDRx)
3328	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3329	else
3330	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3331
3332	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3333	AssertRCReturn(rc, rc);
3334	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3335	AssertRCReturn(rc, rc);
3336
3337	/* The guest's view of its DR7 is unblemished. Use 32-bit write as upper 32-bits MBZ as asserted above. */
3338	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
3339	AssertRCReturn(rc, rc);
3340
3341	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3342	return rc;
3343	}
3344
3345
3346	#ifdef VBOX_STRICT
3347	/**
3348	* Strict function to validate segment registers.
3349	*
3350	* @remarks Requires CR0.
3351	*/
3352	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3353	{
3354	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3355	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3356	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3357	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3358	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3359	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3360	{
3361	/* Protected mode checks */
3362	/* CS */
3363	Assert(pCtx->cs.Attr.n.u1Present);
3364	Assert(!(pCtx->cs.Attr.u & 0xf00));
3365	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3366	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3367	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3368	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3369	\|\| (pCtx->cs.Attr.n.u1Granularity));
3370	/* CS cannot be loaded with NULL in protected mode. */
3371	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3372	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3373	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3374	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3375	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3376	else
3377	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3378	/* SS */
3379	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3380	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3381	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3382	if ( !(pCtx->cr0 & X86_CR0_PE)
3383	\|\| pCtx->cs.Attr.n.u4Type == 3)
3384	{
3385	Assert(!pCtx->ss.Attr.n.u2Dpl);
3386	}
3387	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3388	{
3389	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3390	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3391	Assert(pCtx->ss.Attr.n.u1Present);
3392	Assert(!(pCtx->ss.Attr.u & 0xf00));
3393	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3394	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3395	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3396	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3397	\|\| (pCtx->ss.Attr.n.u1Granularity));
3398	}
3399	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3400	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3401	{
3402	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3403	Assert(pCtx->ds.Attr.n.u1Present);
3404	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3405	Assert(!(pCtx->ds.Attr.u & 0xf00));
3406	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3407	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3408	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3409	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3410	\|\| (pCtx->ds.Attr.n.u1Granularity));
3411	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3412	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3413	}
3414	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3415	{
3416	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3417	Assert(pCtx->es.Attr.n.u1Present);
3418	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3419	Assert(!(pCtx->es.Attr.u & 0xf00));
3420	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3421	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3422	\|\| !(pCtx->es.Attr.n.u1Granularity));
3423	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3424	\|\| (pCtx->es.Attr.n.u1Granularity));
3425	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3426	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3427	}
3428	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3429	{
3430	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3431	Assert(pCtx->fs.Attr.n.u1Present);
3432	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3433	Assert(!(pCtx->fs.Attr.u & 0xf00));
3434	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3435	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3436	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3437	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3438	\|\| (pCtx->fs.Attr.n.u1Granularity));
3439	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3440	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3441	}
3442	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3443	{
3444	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3445	Assert(pCtx->gs.Attr.n.u1Present);
3446	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3447	Assert(!(pCtx->gs.Attr.u & 0xf00));
3448	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3449	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3450	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3451	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3452	\|\| (pCtx->gs.Attr.n.u1Granularity));
3453	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3454	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3455	}
3456	/* 64-bit capable CPUs. */
3457	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3458	Assert(!(pCtx->cs.u64Base >> 32));
3459	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3460	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3461	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3462	# endif
3463	}
3464	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3465	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3466	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3467	{
3468	/* Real and v86 mode checks. */
3469	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3470	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3471	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3472	{
3473	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3474	}
3475	else
3476	{
3477	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3478	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3479	}
3480
3481	/* CS */
3482	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3483	Assert(pCtx->cs.u32Limit == 0xffff);
3484	Assert(u32CSAttr == 0xf3);
3485	/* SS */
3486	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3487	Assert(pCtx->ss.u32Limit == 0xffff);
3488	Assert(u32SSAttr == 0xf3);
3489	/* DS */
3490	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3491	Assert(pCtx->ds.u32Limit == 0xffff);
3492	Assert(u32DSAttr == 0xf3);
3493	/* ES */
3494	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3495	Assert(pCtx->es.u32Limit == 0xffff);
3496	Assert(u32ESAttr == 0xf3);
3497	/* FS */
3498	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3499	Assert(pCtx->fs.u32Limit == 0xffff);
3500	Assert(u32FSAttr == 0xf3);
3501	/* GS */
3502	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3503	Assert(pCtx->gs.u32Limit == 0xffff);
3504	Assert(u32GSAttr == 0xf3);
3505	/* 64-bit capable CPUs. */
3506	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3507	Assert(!(pCtx->cs.u64Base >> 32));
3508	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3509	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3510	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3511	# endif
3512	}
3513	}
3514	#endif /* VBOX_STRICT */
3515
3516
3517	/**
3518	* Writes a guest segment register into the guest-state area in the VMCS.
3519	*
3520	* @returns VBox status code.
3521	* @param pVCpu Pointer to the VMCPU.
3522	* @param idxSel Index of the selector in the VMCS.
3523	* @param idxLimit Index of the segment limit in the VMCS.
3524	* @param idxBase Index of the segment base in the VMCS.
3525	* @param idxAccess Index of the access rights of the segment in the VMCS.
3526	* @param pSelReg Pointer to the segment selector.
3527	* @param pCtx Pointer to the guest-CPU context.
3528	*
3529	* @remarks No-long-jump zone!!!
3530	*/
3531	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3532	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3533	{
3534	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3535	AssertRCReturn(rc, rc);
3536	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3537	AssertRCReturn(rc, rc);
3538	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3539	AssertRCReturn(rc, rc);
3540
3541	uint32_t u32Access = pSelReg->Attr.u;
3542	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3543	{
3544	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3545	u32Access = 0xf3;
3546	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3547	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3548	}
3549	else
3550	{
3551	/*
3552	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3553	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3554	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3555	* loaded in protected-mode have their attribute as 0.
3556	*/
3557	if (!u32Access)
3558	u32Access = X86DESCATTR_UNUSABLE;
3559	}
3560
3561	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3562	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3563	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3564
3565	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3566	AssertRCReturn(rc, rc);
3567	return rc;
3568	}
3569
3570
3571	/**
3572	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3573	* into the guest-state area in the VMCS.
3574	*
3575	* @returns VBox status code.
3576	* @param pVM Pointer to the VM.
3577	* @param pVCPU Pointer to the VMCPU.
3578	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3579	* out-of-sync. Make sure to update the required fields
3580	* before using them.
3581	*
3582	* @remarks Requires CR0 (strict builds validation).
3583	* @remarks No-long-jump zone!!!
3584	*/
3585	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3586	{
3587	int rc = VERR_INTERNAL_ERROR_5;
3588	PVM pVM = pVCpu->CTX_SUFF(pVM);
3589
3590	/*
3591	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3592	*/
3593	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3594	{
3595	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3596	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3597	{
3598	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3599	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3600	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3601	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3602	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3603	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3604	}
3605
3606	#ifdef VBOX_WITH_REM
3607	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3608	{
3609	Assert(pVM->hm.s.vmx.pRealModeTSS);
3610	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3611	if ( pVCpu->hm.s.vmx.fWasInRealMode
3612	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3613	{
3614	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3615	in real-mode (e.g. OpenBSD 4.0) */
3616	REMFlushTBs(pVM);
3617	Log4(("Load: Switch to protected mode detected!\n"));
3618	pVCpu->hm.s.vmx.fWasInRealMode = false;
3619	}
3620	}
3621	#endif
3622	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3623	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3624	AssertRCReturn(rc, rc);
3625	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3626	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3627	AssertRCReturn(rc, rc);
3628	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3629	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3630	AssertRCReturn(rc, rc);
3631	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3632	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3633	AssertRCReturn(rc, rc);
3634	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3635	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3636	AssertRCReturn(rc, rc);
3637	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3638	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3639	AssertRCReturn(rc, rc);
3640
3641	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3642	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3643	#ifdef VBOX_STRICT
3644	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3645	#endif
3646	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3647	}
3648
3649	/*
3650	* Guest TR.
3651	*/
3652	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3653	{
3654	/*
3655	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3656	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3657	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3658	*/
3659	uint16_t u16Sel = 0;
3660	uint32_t u32Limit = 0;
3661	uint64_t u64Base = 0;
3662	uint32_t u32AccessRights = 0;
3663
3664	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3665	{
3666	u16Sel = pMixedCtx->tr.Sel;
3667	u32Limit = pMixedCtx->tr.u32Limit;
3668	u64Base = pMixedCtx->tr.u64Base;
3669	u32AccessRights = pMixedCtx->tr.Attr.u;
3670	}
3671	else
3672	{
3673	Assert(pVM->hm.s.vmx.pRealModeTSS);
3674	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3675
3676	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3677	RTGCPHYS GCPhys;
3678	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3679	AssertRCReturn(rc, rc);
3680
3681	X86DESCATTR DescAttr;
3682	DescAttr.u = 0;
3683	DescAttr.n.u1Present = 1;
3684	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3685
3686	u16Sel = 0;
3687	u32Limit = HM_VTX_TSS_SIZE;
3688	u64Base = GCPhys; /* in real-mode phys = virt. */
3689	u32AccessRights = DescAttr.u;
3690	}
3691
3692	/* Validate. */
3693	Assert(!(u16Sel & RT_BIT(2)));
3694	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3695	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3696	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3697	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3698	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3699	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3700	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3701	Assert( (u32Limit & 0xfff) == 0xfff
3702	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3703	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3704	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3705
3706	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3707	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3708	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3709	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3710
3711	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3712	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3713	}
3714
3715	/*
3716	* Guest GDTR.
3717	*/
3718	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3719	{
3720	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3721	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3722
3723	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3724	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3725	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3726	}
3727
3728	/*
3729	* Guest LDTR.
3730	*/
3731	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3732	{
3733	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3734	uint32_t u32Access = 0;
3735	if (!pMixedCtx->ldtr.Attr.u)
3736	u32Access = X86DESCATTR_UNUSABLE;
3737	else
3738	u32Access = pMixedCtx->ldtr.Attr.u;
3739
3740	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3741	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3742	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3743	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3744
3745	/* Validate. */
3746	if (!(u32Access & X86DESCATTR_UNUSABLE))
3747	{
3748	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3749	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3750	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3751	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3752	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3753	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3754	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3755	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3756	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3757	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3758	}
3759
3760	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3761	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3762	}
3763
3764	/*
3765	* Guest IDTR.
3766	*/
3767	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3768	{
3769	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3770	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3771
3772	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3773	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3774	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3775	}
3776
3777	return VINF_SUCCESS;
3778	}
3779
3780
3781	/**
3782	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3783	* areas. These MSRs will automatically be loaded to the host CPU on every
3784	* successful VM entry and stored from the host CPU on every successful VM exit.
3785	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3786	*
3787	* @returns VBox status code.
3788	* @param pVCpu Pointer to the VMCPU.
3789	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3790	* out-of-sync. Make sure to update the required fields
3791	* before using them.
3792	*
3793	* @remarks No-long-jump zone!!!
3794	*/
3795	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3796	{
3797	AssertPtr(pVCpu);
3798	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3799
3800	/*
3801	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3802	*/
3803	int rc = VINF_SUCCESS;
3804	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3805	{
3806	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3807	PVM pVM = pVCpu->CTX_SUFF(pVM);
3808	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3809	uint32_t cGuestMsrs = 0;
3810
3811	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3812	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3813	* when the guest really is in 64-bit mode. */
3814	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3815	if (fSupportsLongMode)
3816	{
3817	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3818	pGuestMsr->u32Reserved = 0;
3819	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3820	pGuestMsr++; cGuestMsrs++;
3821	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3822	pGuestMsr->u32Reserved = 0;
3823	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3824	pGuestMsr++; cGuestMsrs++;
3825	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3826	pGuestMsr->u32Reserved = 0;
3827	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3828	pGuestMsr++; cGuestMsrs++;
3829	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3830	pGuestMsr->u32Reserved = 0;
3831	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3832	pGuestMsr++; cGuestMsrs++;
3833	}
3834
3835	/*
3836	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3837	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3838	*/
3839	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3840	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3841	{
3842	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3843	pGuestMsr->u32Reserved = 0;
3844	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3845	AssertRCReturn(rc, rc);
3846	pGuestMsr++; cGuestMsrs++;
3847	}
3848
3849	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3850	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3851	{
3852	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3853	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3854	}
3855
3856	/* Update the VCPU's copy of the guest MSR count. */
3857	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3858	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3859	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3860	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3861
3862	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3863	}
3864
3865	/*
3866	* Guest Sysenter MSRs.
3867	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3868	* VM-exits on WRMSRs for these MSRs.
3869	*/
3870	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3871	{
3872	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3873	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3874	}
3875	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3876	{
3877	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3878	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3879	}
3880	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3881	{
3882	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3883	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3884	}
3885
3886	return rc;
3887	}
3888
3889
3890	/**
3891	* Loads the guest activity state into the guest-state area in the VMCS.
3892	*
3893	* @returns VBox status code.
3894	* @param pVCpu Pointer to the VMCPU.
3895	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3896	* out-of-sync. Make sure to update the required fields
3897	* before using them.
3898	*
3899	* @remarks No-long-jump zone!!!
3900	*/
3901	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3902	{
3903	/** @todo See if we can make use of other states, e.g.
3904	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3905	int rc = VINF_SUCCESS;
3906	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3907	{
3908	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3909	AssertRCReturn(rc, rc);
3910	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3911	}
3912	return rc;
3913	}
3914
3915
3916	/**
3917	* Sets up the appropriate function to run guest code.
3918	*
3919	* @returns VBox status code.
3920	* @param pVCpu Pointer to the VMCPU.
3921	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3922	* out-of-sync. Make sure to update the required fields
3923	* before using them.
3924	*
3925	* @remarks No-long-jump zone!!!
3926	*/
3927	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3928	{
3929	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3930	{
3931	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3932	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3933	#endif
3934	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3935	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3936	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3937	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3938	#else
3939	/* 64-bit host or hybrid host. */
3940	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3941	#endif
3942	}
3943	else
3944	{
3945	/* Guest is not in long mode, use the 32-bit handler. */
3946	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3947	}
3948	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3949	return VINF_SUCCESS;
3950	}
3951
3952
3953	/**
3954	* Wrapper for running the guest code in VT-x.
3955	*
3956	* @returns VBox strict status code.
3957	* @param pVM Pointer to the VM.
3958	* @param pVCpu Pointer to the VMCPU.
3959	* @param pCtx Pointer to the guest-CPU context.
3960	*
3961	* @remarks No-long-jump zone!!!
3962	*/
3963	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3964	{
3965	/*
3966	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3967	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3968	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3969	*/
3970	#ifdef VBOX_WITH_KERNEL_USING_XMM
3971	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3972	#else
3973	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3974	#endif
3975	}
3976
3977
3978	/**
3979	* Reports world-switch error and dumps some useful debug info.
3980	*
3981	* @param pVM Pointer to the VM.
3982	* @param pVCpu Pointer to the VMCPU.
3983	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3984	* @param pCtx Pointer to the guest-CPU context.
3985	* @param pVmxTransient Pointer to the VMX transient structure (only
3986	* exitReason updated).
3987	*/
3988	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3989	{
3990	Assert(pVM);
3991	Assert(pVCpu);
3992	Assert(pCtx);
3993	Assert(pVmxTransient);
3994	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3995
3996	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
3997	switch (rcVMRun)
3998	{
3999	case VERR_VMX_INVALID_VMXON_PTR:
4000	AssertFailed();
4001	break;
4002	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4003	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4004	{
4005	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4006	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4007	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4008	AssertRC(rc);
4009
4010	#ifdef VBOX_STRICT
4011	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4012	pVmxTransient->uExitReason));
4013	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4014	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4015	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4016	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4017	else
4018	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4019
4020	/* VMX control bits. */
4021	uint32_t u32Val;
4022	uint64_t u64Val;
4023	HMVMXHCUINTREG uHCReg;
4024	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4025	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4026	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4027	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4028	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4029	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4030	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4031	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4032	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4033	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4034	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4035	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4036	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4037	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4038	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4039	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4040	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4041	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4042	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4043	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4044	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4045	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4046	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4047	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4048	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4049	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4050	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4051	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4052	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4053	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4054	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4055	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4056	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4057	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4058	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4059	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4060	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4061	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4062	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4063	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4064	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4065	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4066
4067	/* Guest bits. */
4068	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4069	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4070	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4071	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4072	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4073	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4074	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4075	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4076
4077	/* Host bits. */
4078	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4079	Log4(("Host CR0 %#RHr\n", uHCReg));
4080	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4081	Log4(("Host CR3 %#RHr\n", uHCReg));
4082	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4083	Log4(("Host CR4 %#RHr\n", uHCReg));
4084
4085	RTGDTR HostGdtr;
4086	PCX86DESCHC pDesc;
4087	ASMGetGDTR(&HostGdtr);
4088	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4089	Log4(("Host CS %#08x\n", u32Val));
4090	if (u32Val < HostGdtr.cbGdt)
4091	{
4092	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4093	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4094	}
4095
4096	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4097	Log4(("Host DS %#08x\n", u32Val));
4098	if (u32Val < HostGdtr.cbGdt)
4099	{
4100	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4101	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4102	}
4103
4104	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4105	Log4(("Host ES %#08x\n", u32Val));
4106	if (u32Val < HostGdtr.cbGdt)
4107	{
4108	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4109	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4110	}
4111
4112	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4113	Log4(("Host FS %#08x\n", u32Val));
4114	if (u32Val < HostGdtr.cbGdt)
4115	{
4116	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4117	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4118	}
4119
4120	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4121	Log4(("Host GS %#08x\n", u32Val));
4122	if (u32Val < HostGdtr.cbGdt)
4123	{
4124	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4125	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4126	}
4127
4128	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4129	Log4(("Host SS %#08x\n", u32Val));
4130	if (u32Val < HostGdtr.cbGdt)
4131	{
4132	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4133	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4134	}
4135
4136	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4137	Log4(("Host TR %#08x\n", u32Val));
4138	if (u32Val < HostGdtr.cbGdt)
4139	{
4140	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4141	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4142	}
4143
4144	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4145	Log4(("Host TR Base %#RHv\n", uHCReg));
4146	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4147	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4148	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4149	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4150	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4151	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4152	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4153	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4154	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4155	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4156	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4157	Log4(("Host RSP %#RHv\n", uHCReg));
4158	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4159	Log4(("Host RIP %#RHv\n", uHCReg));
4160	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4161	if (HMVMX_IS_64BIT_HOST_MODE())
4162	{
4163	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4164	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4165	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4166	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4167	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4168	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4169	}
4170	# endif
4171	#endif /* VBOX_STRICT */
4172	break;
4173	}
4174
4175	default:
4176	/* Impossible */
4177	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4178	break;
4179	}
4180	NOREF(pVM);
4181	}
4182
4183
4184	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4185	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4186	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4187	#endif
4188	#ifdef VBOX_STRICT
4189	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4190	{
4191	switch (idxField)
4192	{
4193	case VMX_VMCS_GUEST_RIP:
4194	case VMX_VMCS_GUEST_RSP:
4195	case VMX_VMCS_GUEST_SYSENTER_EIP:
4196	case VMX_VMCS_GUEST_SYSENTER_ESP:
4197	case VMX_VMCS_GUEST_GDTR_BASE:
4198	case VMX_VMCS_GUEST_IDTR_BASE:
4199	case VMX_VMCS_GUEST_CS_BASE:
4200	case VMX_VMCS_GUEST_DS_BASE:
4201	case VMX_VMCS_GUEST_ES_BASE:
4202	case VMX_VMCS_GUEST_FS_BASE:
4203	case VMX_VMCS_GUEST_GS_BASE:
4204	case VMX_VMCS_GUEST_SS_BASE:
4205	case VMX_VMCS_GUEST_LDTR_BASE:
4206	case VMX_VMCS_GUEST_TR_BASE:
4207	case VMX_VMCS_GUEST_CR3:
4208	return true;
4209	}
4210	return false;
4211	}
4212
4213	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4214	{
4215	switch (idxField)
4216	{
4217	/* Read-only fields. */
4218	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4219	return true;
4220	}
4221	/* Remaining readable fields should also be writable. */
4222	return hmR0VmxIsValidWriteField(idxField);
4223	}
4224	#endif /* VBOX_STRICT */
4225
4226
4227	/**
4228	* Executes the specified handler in 64-bit mode.
4229	*
4230	* @returns VBox status code.
4231	* @param pVM Pointer to the VM.
4232	* @param pVCpu Pointer to the VMCPU.
4233	* @param pCtx Pointer to the guest CPU context.
4234	* @param enmOp The operation to perform.
4235	* @param cbParam Number of parameters.
4236	* @param paParam Array of 32-bit parameters.
4237	*/
4238	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4239	uint32_t *paParam)
4240	{
4241	int rc, rc2;
4242	PHMGLOBLCPUINFO pCpu;
4243	RTHCPHYS HCPhysCpuPage;
4244	RTCCUINTREG uOldEFlags;
4245
4246	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4247	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4248	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4249	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4250
4251	#ifdef VBOX_STRICT
4252	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4253	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4254
4255	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4256	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4257	#endif
4258
4259	/* Disable interrupts. */
4260	uOldEFlags = ASMIntDisableFlags();
4261
4262	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4263	RTCPUID idHostCpu = RTMpCpuId();
4264	CPUMR0SetLApic(pVM, idHostCpu);
4265	#endif
4266
4267	pCpu = HMR0GetCurrentCpu();
4268	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4269
4270	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4271	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4272
4273	/* Leave VMX Root Mode. */
4274	VMXDisable();
4275
4276	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4277
4278	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4279	CPUMSetHyperEIP(pVCpu, enmOp);
4280	for (int i = (int)cbParam - 1; i >= 0; i--)
4281	CPUMPushHyper(pVCpu, paParam[i]);
4282
4283	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4284
4285	/* Call the switcher. */
4286	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4287	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4288
4289	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4290	/* Make sure the VMX instructions don't cause #UD faults. */
4291	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4292
4293	/* Re-enter VMX Root Mode */
4294	rc2 = VMXEnable(HCPhysCpuPage);
4295	if (RT_FAILURE(rc2))
4296	{
4297	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4298	ASMSetFlags(uOldEFlags);
4299	return rc2;
4300	}
4301
4302	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4303	AssertRC(rc2);
4304	Assert(!(ASMGetFlags() & X86_EFL_IF));
4305	ASMSetFlags(uOldEFlags);
4306	return rc;
4307	}
4308
4309
4310	/**
4311	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4312	* supporting 64-bit guests.
4313	*
4314	* @returns VBox status code.
4315	* @param fResume Whether to VMLAUNCH or VMRESUME.
4316	* @param pCtx Pointer to the guest-CPU context.
4317	* @param pCache Pointer to the VMCS cache.
4318	* @param pVM Pointer to the VM.
4319	* @param pVCpu Pointer to the VMCPU.
4320	*/
4321	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4322	{
4323	uint32_t aParam[6];
4324	PHMGLOBLCPUINFO pCpu = NULL;
4325	RTHCPHYS HCPhysCpuPage = 0;
4326	int rc = VERR_INTERNAL_ERROR_5;
4327
4328	pCpu = HMR0GetCurrentCpu();
4329	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4330
4331	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4332	pCache->uPos = 1;
4333	pCache->interPD = PGMGetInterPaeCR3(pVM);
4334	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4335	#endif
4336
4337	#ifdef VBOX_STRICT
4338	pCache->TestIn.HCPhysCpuPage = 0;
4339	pCache->TestIn.HCPhysVmcs = 0;
4340	pCache->TestIn.pCache = 0;
4341	pCache->TestOut.HCPhysVmcs = 0;
4342	pCache->TestOut.pCache = 0;
4343	pCache->TestOut.pCtx = 0;
4344	pCache->TestOut.eflags = 0;
4345	#endif
4346
4347	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4348	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4349	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4350	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4351	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4352	aParam[5] = 0;
4353
4354	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4355	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4356	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4357	#endif
4358	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4359
4360	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4361	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4362	Assert(pCtx->dr[4] == 10);
4363	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4364	#endif
4365
4366	#ifdef VBOX_STRICT
4367	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4368	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4369	pVCpu->hm.s.vmx.HCPhysVmcs));
4370	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4371	pCache->TestOut.HCPhysVmcs));
4372	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4373	pCache->TestOut.pCache));
4374	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4375	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4376	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4377	pCache->TestOut.pCtx));
4378	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4379	#endif
4380	return rc;
4381	}
4382
4383
4384	/**
4385	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4386	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4387	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4388	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4389	*
4390	* @returns VBox status code.
4391	* @param pVM Pointer to the VM.
4392	* @param pVCpu Pointer to the VMCPU.
4393	*/
4394	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4395	{
4396	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4397	{ \
4398	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4399	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4400	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4401	++cReadFields; \
4402	}
4403
4404	AssertPtr(pVM);
4405	AssertPtr(pVCpu);
4406	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4407	uint32_t cReadFields = 0;
4408
4409	/*
4410	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4411	* and serve to indicate exceptions to the rules.
4412	*/
4413
4414	/* Guest-natural selector base fields. */
4415	#if 0
4416	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4417	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4418	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4419	#endif
4420	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4421	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4422	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4423	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4424	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4425	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4426	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4427	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4428	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4429	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4430	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4431	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4432	#if 0
4433	/* Unused natural width guest-state fields. */
4434	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4435	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4436	#endif
4437	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4438	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4439
4440	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4441	#if 0
4442	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4443	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4444	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4445	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4446	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4447	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4448	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4449	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4450	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4451	#endif
4452
4453	/* Natural width guest-state fields. */
4454	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4455	#if 0
4456	/* Currently unused field. */
4457	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4458	#endif
4459
4460	if (pVM->hm.s.fNestedPaging)
4461	{
4462	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4463	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4464	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4465	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4466	}
4467	else
4468	{
4469	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4470	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4471	}
4472
4473	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4474	return VINF_SUCCESS;
4475	}
4476
4477
4478	/**
4479	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4480	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4481	* darwin, running 64-bit guests).
4482	*
4483	* @returns VBox status code.
4484	* @param pVCpu Pointer to the VMCPU.
4485	* @param idxField The VMCS field encoding.
4486	* @param u64Val 16, 32 or 64 bits value.
4487	*/
4488	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4489	{
4490	int rc;
4491	switch (idxField)
4492	{
4493	/*
4494	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4495	*/
4496	/* 64-bit Control fields. */
4497	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4498	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4499	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4500	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4501	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4502	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4503	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4504	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4505	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4506	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4507	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4508	case VMX_VMCS64_CTRL_EPTP_FULL:
4509	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4510	/* 64-bit Guest-state fields. */
4511	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4512	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4513	case VMX_VMCS64_GUEST_PAT_FULL:
4514	case VMX_VMCS64_GUEST_EFER_FULL:
4515	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4516	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4517	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4518	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4519	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4520	/* 64-bit Host-state fields. */
4521	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4522	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4523	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4524	{
4525	rc = VMXWriteVmcs32(idxField, u64Val);
4526	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4527	break;
4528	}
4529
4530	/*
4531	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4532	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4533	*/
4534	/* Natural-width Guest-state fields. */
4535	case VMX_VMCS_GUEST_CR3:
4536	case VMX_VMCS_GUEST_ES_BASE:
4537	case VMX_VMCS_GUEST_CS_BASE:
4538	case VMX_VMCS_GUEST_SS_BASE:
4539	case VMX_VMCS_GUEST_DS_BASE:
4540	case VMX_VMCS_GUEST_FS_BASE:
4541	case VMX_VMCS_GUEST_GS_BASE:
4542	case VMX_VMCS_GUEST_LDTR_BASE:
4543	case VMX_VMCS_GUEST_TR_BASE:
4544	case VMX_VMCS_GUEST_GDTR_BASE:
4545	case VMX_VMCS_GUEST_IDTR_BASE:
4546	case VMX_VMCS_GUEST_RSP:
4547	case VMX_VMCS_GUEST_RIP:
4548	case VMX_VMCS_GUEST_SYSENTER_ESP:
4549	case VMX_VMCS_GUEST_SYSENTER_EIP:
4550	{
4551	if (!(u64Val >> 32))
4552	{
4553	/* If this field is 64-bit, VT-x will zero out the top bits. */
4554	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4555	}
4556	else
4557	{
4558	/* Assert that only the 32->64 switcher case should ever come here. */
4559	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4560	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4561	}
4562	break;
4563	}
4564
4565	default:
4566	{
4567	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4568	rc = VERR_INVALID_PARAMETER;
4569	break;
4570	}
4571	}
4572	AssertRCReturn(rc, rc);
4573	return rc;
4574	}
4575
4576
4577	/**
4578	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4579	* hosts (except darwin) for 64-bit guests.
4580	*
4581	* @param pVCpu Pointer to the VMCPU.
4582	* @param idxField The VMCS field encoding.
4583	* @param u64Val 16, 32 or 64 bits value.
4584	*/
4585	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4586	{
4587	AssertPtr(pVCpu);
4588	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4589
4590	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4591	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4592
4593	/* Make sure there are no duplicates. */
4594	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4595	{
4596	if (pCache->Write.aField[i] == idxField)
4597	{
4598	pCache->Write.aFieldVal[i] = u64Val;
4599	return VINF_SUCCESS;
4600	}
4601	}
4602
4603	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4604	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4605	pCache->Write.cValidEntries++;
4606	return VINF_SUCCESS;
4607	}
4608
4609	/* Enable later when the assembly code uses these as callbacks. */
4610	#if 0
4611	/*
4612	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4613	*
4614	* @param pVCpu Pointer to the VMCPU.
4615	* @param pCache Pointer to the VMCS cache.
4616	*
4617	* @remarks No-long-jump zone!!!
4618	*/
4619	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4620	{
4621	AssertPtr(pCache);
4622	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4623	{
4624	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4625	AssertRC(rc);
4626	}
4627	pCache->Write.cValidEntries = 0;
4628	}
4629
4630
4631	/**
4632	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4633	*
4634	* @param pVCpu Pointer to the VMCPU.
4635	* @param pCache Pointer to the VMCS cache.
4636	*
4637	* @remarks No-long-jump zone!!!
4638	*/
4639	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4640	{
4641	AssertPtr(pCache);
4642	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4643	{
4644	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4645	AssertRC(rc);
4646	}
4647	}
4648	#endif
4649	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4650
4651
4652	/**
4653	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4654	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4655	* timer.
4656	*
4657	* @returns VBox status code.
4658	* @param pVCpu Pointer to the VMCPU.
4659	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4660	* out-of-sync. Make sure to update the required fields
4661	* before using them.
4662	* @remarks No-long-jump zone!!!
4663	*/
4664	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4665	{
4666	int rc = VERR_INTERNAL_ERROR_5;
4667	bool fOffsettedTsc = false;
4668	PVM pVM = pVCpu->CTX_SUFF(pVM);
4669	if (pVM->hm.s.vmx.fUsePreemptTimer)
4670	{
4671	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4672
4673	/* Make sure the returned values have sane upper and lower boundaries. */
4674	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4675	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4676	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4677	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4678
4679	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4680	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4681	}
4682	else
4683	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4684
4685	if (fOffsettedTsc)
4686	{
4687	uint64_t u64CurTSC = ASMReadTSC();
4688	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4689	{
4690	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4691	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4692
4693	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4694	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4695	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4696	}
4697	else
4698	{
4699	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4700	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4701	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4702	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4703	}
4704	}
4705	else
4706	{
4707	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4708	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4709	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4710	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4711	}
4712	}
4713
4714
4715	/**
4716	* Determines if an exception is a contributory exception. Contributory
4717	* exceptions are ones which can cause double-faults. Page-fault is
4718	* intentionally not included here as it's a conditional contributory exception.
4719	*
4720	* @returns true if the exception is contributory, false otherwise.
4721	* @param uVector The exception vector.
4722	*/
4723	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4724	{
4725	switch (uVector)
4726	{
4727	case X86_XCPT_GP:
4728	case X86_XCPT_SS:
4729	case X86_XCPT_NP:
4730	case X86_XCPT_TS:
4731	case X86_XCPT_DE:
4732	return true;
4733	default:
4734	break;
4735	}
4736	return false;
4737	}
4738
4739
4740	/**
4741	* Sets an event as a pending event to be injected into the guest.
4742	*
4743	* @param pVCpu Pointer to the VMCPU.
4744	* @param u32IntrInfo The VM-entry interruption-information field.
4745	* @param cbInstr The VM-entry instruction length in bytes (for software
4746	* interrupts, exceptions and privileged software
4747	* exceptions).
4748	* @param u32ErrCode The VM-entry exception error code.
4749	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4750	* page-fault.
4751	*
4752	* @remarks Statistics counter assumes this is a guest event being injected or
4753	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4754	* always incremented.
4755	*/
4756	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4757	RTGCUINTPTR GCPtrFaultAddress)
4758	{
4759	Assert(!pVCpu->hm.s.Event.fPending);
4760	pVCpu->hm.s.Event.fPending = true;
4761	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4762	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4763	pVCpu->hm.s.Event.cbInstr = cbInstr;
4764	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4765
4766	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4767	}
4768
4769
4770	/**
4771	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4772	*
4773	* @param pVCpu Pointer to the VMCPU.
4774	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4775	* out-of-sync. Make sure to update the required fields
4776	* before using them.
4777	*/
4778	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4779	{
4780	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4781	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4782	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4783	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4784	}
4785
4786
4787	/**
4788	* Handle a condition that occurred while delivering an event through the guest
4789	* IDT.
4790	*
4791	* @returns VBox status code (informational error codes included).
4792	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4793	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4794	* continue execution of the guest which will delivery the #DF.
4795	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4796	*
4797	* @param pVCpu Pointer to the VMCPU.
4798	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4799	* out-of-sync. Make sure to update the required fields
4800	* before using them.
4801	* @param pVmxTransient Pointer to the VMX transient structure.
4802	*
4803	* @remarks No-long-jump zone!!!
4804	*/
4805	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4806	{
4807	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4808	AssertRC(rc);
4809	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4810	{
4811	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4812	AssertRCReturn(rc, rc);
4813
4814	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4815	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4816	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4817
4818	typedef enum
4819	{
4820	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4821	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4822	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4823	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4824	} VMXREFLECTXCPT;
4825
4826	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4827	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4828	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4829	{
4830	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4831	{
4832	enmReflect = VMXREFLECTXCPT_XCPT;
4833	#ifdef VBOX_STRICT
4834	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4835	&& uExitVector == X86_XCPT_PF)
4836	{
4837	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4838	}
4839	#endif
4840	if ( uExitVector == X86_XCPT_PF
4841	&& uIdtVector == X86_XCPT_PF)
4842	{
4843	pVmxTransient->fVectoringPF = true;
4844	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4845	}
4846	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4847	&& hmR0VmxIsContributoryXcpt(uExitVector)
4848	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4849	\|\| uIdtVector == X86_XCPT_PF))
4850	{
4851	enmReflect = VMXREFLECTXCPT_DF;
4852	}
4853	else if (uIdtVector == X86_XCPT_DF)
4854	enmReflect = VMXREFLECTXCPT_TF;
4855	}
4856	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4857	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4858	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4859	{
4860	/*
4861	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4862	* (whatever they are) as they reoccur when restarting the instruction.
4863	*/
4864	enmReflect = VMXREFLECTXCPT_XCPT;
4865	}
4866	}
4867	else
4868	{
4869	/*
4870	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
4871	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
4872	* original exception to the guest after handling the VM-exit.
4873	*/
4874	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4875	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4876	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4877	{
4878	enmReflect = VMXREFLECTXCPT_XCPT;
4879	}
4880	}
4881
4882	switch (enmReflect)
4883	{
4884	case VMXREFLECTXCPT_XCPT:
4885	{
4886	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4887	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4888	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
4889
4890	uint32_t u32ErrCode = 0;
4891	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
4892	{
4893	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4894	AssertRCReturn(rc, rc);
4895	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4896	}
4897
4898	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4899	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4900	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4901	rc = VINF_SUCCESS;
4902	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
4903	pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4904
4905	break;
4906	}
4907
4908	case VMXREFLECTXCPT_DF:
4909	{
4910	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4911	rc = VINF_HM_DOUBLE_FAULT;
4912	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
4913	pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4914
4915	break;
4916	}
4917
4918	case VMXREFLECTXCPT_TF:
4919	{
4920	rc = VINF_EM_RESET;
4921	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
4922	uExitVector));
4923	break;
4924	}
4925
4926	default:
4927	Assert(rc == VINF_SUCCESS);
4928	break;
4929	}
4930	}
4931	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4932	return rc;
4933	}
4934
4935
4936	/**
4937	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4938	*
4939	* @returns VBox status code.
4940	* @param pVCpu Pointer to the VMCPU.
4941	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4942	* out-of-sync. Make sure to update the required fields
4943	* before using them.
4944	*
4945	* @remarks No-long-jump zone!!!
4946	*/
4947	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4948	{
4949	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4950	{
4951	uint32_t uVal = 0;
4952	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4953	AssertRCReturn(rc, rc);
4954	uint32_t uShadow = 0;
4955	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4956	AssertRCReturn(rc, rc);
4957
4958	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
4959	CPUMSetGuestCR0(pVCpu, uVal);
4960	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4961	}
4962	return VINF_SUCCESS;
4963	}
4964
4965
4966	/**
4967	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4968	*
4969	* @returns VBox status code.
4970	* @param pVCpu Pointer to the VMCPU.
4971	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4972	* out-of-sync. Make sure to update the required fields
4973	* before using them.
4974	*
4975	* @remarks No-long-jump zone!!!
4976	*/
4977	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4978	{
4979	int rc = VINF_SUCCESS;
4980	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4981	{
4982	uint32_t uVal = 0;
4983	uint32_t uShadow = 0;
4984	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4985	AssertRCReturn(rc, rc);
4986	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4987	AssertRCReturn(rc, rc);
4988
4989	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
4990	CPUMSetGuestCR4(pVCpu, uVal);
4991	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4992	}
4993	return rc;
4994	}
4995
4996
4997	/**
4998	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4999	*
5000	* @returns VBox status code.
5001	* @param pVCpu Pointer to the VMCPU.
5002	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5003	* out-of-sync. Make sure to update the required fields
5004	* before using them.
5005	*
5006	* @remarks No-long-jump zone!!!
5007	*/
5008	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5009	{
5010	int rc = VINF_SUCCESS;
5011	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5012	{
5013	uint64_t u64Val = 0;
5014	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5015	AssertRCReturn(rc, rc);
5016
5017	pMixedCtx->rip = u64Val;
5018	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5019	}
5020	return rc;
5021	}
5022
5023
5024	/**
5025	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5026	*
5027	* @returns VBox status code.
5028	* @param pVCpu Pointer to the VMCPU.
5029	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5030	* out-of-sync. Make sure to update the required fields
5031	* before using them.
5032	*
5033	* @remarks No-long-jump zone!!!
5034	*/
5035	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5036	{
5037	int rc = VINF_SUCCESS;
5038	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5039	{
5040	uint64_t u64Val = 0;
5041	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5042	AssertRCReturn(rc, rc);
5043
5044	pMixedCtx->rsp = u64Val;
5045	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5046	}
5047	return rc;
5048	}
5049
5050
5051	/**
5052	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5053	*
5054	* @returns VBox status code.
5055	* @param pVCpu Pointer to the VMCPU.
5056	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5057	* out-of-sync. Make sure to update the required fields
5058	* before using them.
5059	*
5060	* @remarks No-long-jump zone!!!
5061	*/
5062	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5063	{
5064	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5065	{
5066	uint32_t uVal = 0;
5067	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5068	AssertRCReturn(rc, rc);
5069
5070	pMixedCtx->eflags.u32 = uVal;
5071	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5072	{
5073	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5074	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5075
5076	pMixedCtx->eflags.Bits.u1VM = 0;
5077	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
5078	}
5079
5080	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5081	}
5082	return VINF_SUCCESS;
5083	}
5084
5085
5086	/**
5087	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5088	* guest-CPU context.
5089	*/
5090	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5091	{
5092	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5093	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5094	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5095	return rc;
5096	}
5097
5098
5099	/**
5100	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5101	* from the guest-state area in the VMCS.
5102	*
5103	* @param pVCpu Pointer to the VMCPU.
5104	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5105	* out-of-sync. Make sure to update the required fields
5106	* before using them.
5107	*
5108	* @remarks No-long-jump zone!!!
5109	*/
5110	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5111	{
5112	uint32_t uIntrState = 0;
5113	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5114	AssertRC(rc);
5115
5116	if (!uIntrState)
5117	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5118	else
5119	{
5120	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5121	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5122	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5123	AssertRC(rc);
5124	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5125	AssertRC(rc);
5126
5127	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5128	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5129	}
5130	}
5131
5132
5133	/**
5134	* Saves the guest's activity state.
5135	*
5136	* @returns VBox status code.
5137	* @param pVCpu Pointer to the VMCPU.
5138	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5139	* out-of-sync. Make sure to update the required fields
5140	* before using them.
5141	*
5142	* @remarks No-long-jump zone!!!
5143	*/
5144	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5145	{
5146	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5147	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5148	return VINF_SUCCESS;
5149	}
5150
5151
5152	/**
5153	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5154	* the current VMCS into the guest-CPU context.
5155	*
5156	* @returns VBox status code.
5157	* @param pVCpu Pointer to the VMCPU.
5158	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5159	* out-of-sync. Make sure to update the required fields
5160	* before using them.
5161	*
5162	* @remarks No-long-jump zone!!!
5163	*/
5164	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5165	{
5166	int rc = VINF_SUCCESS;
5167	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5168	{
5169	uint32_t u32Val = 0;
5170	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5171	pMixedCtx->SysEnter.cs = u32Val;
5172	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5173	}
5174
5175	uint64_t u64Val = 0;
5176	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5177	{
5178	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5179	pMixedCtx->SysEnter.eip = u64Val;
5180	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5181	}
5182	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5183	{
5184	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5185	pMixedCtx->SysEnter.esp = u64Val;
5186	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5187	}
5188	return rc;
5189	}
5190
5191
5192	/**
5193	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5194	* context.
5195	*
5196	* @returns VBox status code.
5197	* @param pVCpu Pointer to the VMCPU.
5198	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5199	* out-of-sync. Make sure to update the required fields
5200	* before using them.
5201	*
5202	* @remarks No-long-jump zone!!!
5203	*/
5204	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5205	{
5206	int rc = VINF_SUCCESS;
5207	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5208	{
5209	uint64_t u64Val = 0;
5210	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5211	pMixedCtx->fs.u64Base = u64Val;
5212	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5213	}
5214	return rc;
5215	}
5216
5217
5218	/**
5219	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5220	* context.
5221	*
5222	* @returns VBox status code.
5223	* @param pVCpu Pointer to the VMCPU.
5224	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5225	* out-of-sync. Make sure to update the required fields
5226	* before using them.
5227	*
5228	* @remarks No-long-jump zone!!!
5229	*/
5230	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5231	{
5232	int rc = VINF_SUCCESS;
5233	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5234	{
5235	uint64_t u64Val = 0;
5236	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5237	pMixedCtx->gs.u64Base = u64Val;
5238	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5239	}
5240	return rc;
5241	}
5242
5243
5244	/**
5245	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5246	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5247	* and TSC_AUX.
5248	*
5249	* @returns VBox status code.
5250	* @param pVCpu Pointer to the VMCPU.
5251	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5252	* out-of-sync. Make sure to update the required fields
5253	* before using them.
5254	*
5255	* @remarks No-long-jump zone!!!
5256	*/
5257	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5258	{
5259	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5260	return VINF_SUCCESS;
5261
5262	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5263	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5264	{
5265	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5266	pMsr += i;
5267	switch (pMsr->u32IndexMSR)
5268	{
5269	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5270	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5271	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5272	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5273	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5274	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5275	default:
5276	{
5277	AssertFailed();
5278	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5279	}
5280	}
5281	}
5282	#endif
5283
5284	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5285	return VINF_SUCCESS;
5286	}
5287
5288
5289	/**
5290	* Saves the guest control registers from the current VMCS into the guest-CPU
5291	* context.
5292	*
5293	* @returns VBox status code.
5294	* @param pVCpu Pointer to the VMCPU.
5295	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5296	* out-of-sync. Make sure to update the required fields
5297	* before using them.
5298	*
5299	* @remarks No-long-jump zone!!!
5300	*/
5301	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5302	{
5303	/* Guest CR0. Guest FPU. */
5304	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5305	AssertRCReturn(rc, rc);
5306
5307	/* Guest CR4. */
5308	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5309	AssertRCReturn(rc, rc);
5310
5311	/* Guest CR2 - updated always during the world-switch or in #PF. */
5312	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5313	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5314	{
5315	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5316	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5317
5318	PVM pVM = pVCpu->CTX_SUFF(pVM);
5319	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5320	\|\| ( pVM->hm.s.fNestedPaging
5321	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5322	{
5323	uint64_t u64Val = 0;
5324	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5325	if (pMixedCtx->cr3 != u64Val)
5326	{
5327	CPUMSetGuestCR3(pVCpu, u64Val);
5328	if (VMMRZCallRing3IsEnabled(pVCpu))
5329	{
5330	PGMUpdateCR3(pVCpu, u64Val);
5331	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5332	}
5333	else
5334	{
5335	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5336	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5337	}
5338	}
5339
5340	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5341	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5342	{
5343	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5344	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5345	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5346	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5347
5348	if (VMMRZCallRing3IsEnabled(pVCpu))
5349	{
5350	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5351	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5352	}
5353	else
5354	{
5355	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5356	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5357	}
5358	}
5359	}
5360
5361	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5362	}
5363
5364	/*
5365	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5366	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5367	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5368	*
5369	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5370	*/
5371	if (VMMRZCallRing3IsEnabled(pVCpu))
5372	{
5373	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5374	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5375
5376	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5377	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5378
5379	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5380	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5381	}
5382
5383	return rc;
5384	}
5385
5386
5387	/**
5388	* Reads a guest segment register from the current VMCS into the guest-CPU
5389	* context.
5390	*
5391	* @returns VBox status code.
5392	* @param pVCpu Pointer to the VMCPU.
5393	* @param idxSel Index of the selector in the VMCS.
5394	* @param idxLimit Index of the segment limit in the VMCS.
5395	* @param idxBase Index of the segment base in the VMCS.
5396	* @param idxAccess Index of the access rights of the segment in the VMCS.
5397	* @param pSelReg Pointer to the segment selector.
5398	*
5399	* @remarks No-long-jump zone!!!
5400	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5401	* macro as that takes care of whether to read from the VMCS cache or
5402	* not.
5403	*/
5404	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5405	PCPUMSELREG pSelReg)
5406	{
5407	uint32_t u32Val = 0;
5408	int rc = VMXReadVmcs32(idxSel, &u32Val);
5409	AssertRCReturn(rc, rc);
5410	pSelReg->Sel = (uint16_t)u32Val;
5411	pSelReg->ValidSel = (uint16_t)u32Val;
5412	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5413
5414	rc = VMXReadVmcs32(idxLimit, &u32Val);
5415	AssertRCReturn(rc, rc);
5416	pSelReg->u32Limit = u32Val;
5417
5418	uint64_t u64Val = 0;
5419	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5420	AssertRCReturn(rc, rc);
5421	pSelReg->u64Base = u64Val;
5422
5423	rc = VMXReadVmcs32(idxAccess, &u32Val);
5424	AssertRCReturn(rc, rc);
5425	pSelReg->Attr.u = u32Val;
5426
5427	/*
5428	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5429	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5430	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5431	*
5432	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5433	*
5434	* bird: This isn't quite as simple. VT-x and VBox(!) requires the DPL for SS to be the same as CPL. In 64-bit mode it
5435	* is possible (int/trap/xxx injects does this when switching rings) to load SS with a NULL selector and RPL=CPL.
5436	* The Attr.u = X86DESCATTR_UNUSABLE works fine as long as nobody uses ring-1 or ring-2. VT-x updates the DPL
5437	* correctly in the attributes of SS even when the unusable bit is set, we need to preserve the DPL or we get invalid
5438	* guest state trouble. Try bs2-cpu-hidden-regs-1.
5439	*/
5440	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5441	{
5442	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5443	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x\n", idxSel, pSelReg->Attr.u));
5444
5445	if (idxSel == VMX_VMCS16_GUEST_FIELD_SS)
5446	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_DPL;
5447	else if (idxSel == VMX_VMCS16_GUEST_FIELD_CS)
5448	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G;
5449	else
5450	pSelReg->Attr.u = X86DESCATTR_UNUSABLE;
5451	}
5452	return VINF_SUCCESS;
5453	}
5454
5455
5456	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5457	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5458	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5459	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5460	#else
5461	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5462	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5463	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5464	#endif
5465
5466
5467	/**
5468	* Saves the guest segment registers from the current VMCS into the guest-CPU
5469	* context.
5470	*
5471	* @returns VBox status code.
5472	* @param pVCpu Pointer to the VMCPU.
5473	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5474	* out-of-sync. Make sure to update the required fields
5475	* before using them.
5476	*
5477	* @remarks No-long-jump zone!!!
5478	*/
5479	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5480	{
5481	/* Guest segment registers. */
5482	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5483	{
5484	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5485	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5486	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5487	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5488	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5489	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5490	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5491
5492	/* Restore segment attributes for real-on-v86 mode hack. */
5493	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5494	{
5495	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5496	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5497	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5498	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5499	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5500	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5501	}
5502	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5503	}
5504
5505	return VINF_SUCCESS;
5506	}
5507
5508
5509	/**
5510	* Saves the guest descriptor table registers and task register from the current
5511	* VMCS into the guest-CPU context.
5512	*
5513	* @returns VBox status code.
5514	* @param pVCpu Pointer to the VMCPU.
5515	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5516	* out-of-sync. Make sure to update the required fields
5517	* before using them.
5518	*
5519	* @remarks No-long-jump zone!!!
5520	*/
5521	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5522	{
5523	int rc = VINF_SUCCESS;
5524
5525	/* Guest LDTR. */
5526	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5527	{
5528	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5529	AssertRCReturn(rc, rc);
5530	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5531	}
5532
5533	/* Guest GDTR. */
5534	uint64_t u64Val = 0;
5535	uint32_t u32Val = 0;
5536	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5537	{
5538	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5539	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5540	pMixedCtx->gdtr.pGdt = u64Val;
5541	pMixedCtx->gdtr.cbGdt = u32Val;
5542	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5543	}
5544
5545	/* Guest IDTR. */
5546	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5547	{
5548	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5549	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5550	pMixedCtx->idtr.pIdt = u64Val;
5551	pMixedCtx->idtr.cbIdt = u32Val;
5552	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5553	}
5554
5555	/* Guest TR. */
5556	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5557	{
5558	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5559	AssertRCReturn(rc, rc);
5560
5561	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5562	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5563	{
5564	rc = VMXLOCAL_READ_SEG(TR, tr);
5565	AssertRCReturn(rc, rc);
5566	}
5567	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5568	}
5569	return rc;
5570	}
5571
5572	#undef VMXLOCAL_READ_SEG
5573
5574
5575	/**
5576	* Saves the guest debug registers from the current VMCS into the guest-CPU
5577	* context.
5578	*
5579	* @returns VBox status code.
5580	* @param pVCpu Pointer to the VMCPU.
5581	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5582	* out-of-sync. Make sure to update the required fields
5583	* before using them.
5584	*
5585	* @remarks No-long-jump zone!!!
5586	*/
5587	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5588	{
5589	int rc = VINF_SUCCESS;
5590	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5591	{
5592	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5593	uint32_t u32Val;
5594	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5595	pMixedCtx->dr[7] = u32Val;
5596
5597	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5598	}
5599	return rc;
5600	}
5601
5602
5603	/**
5604	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
5605	*
5606	* @returns VBox status code.
5607	* @param pVCpu Pointer to the VMCPU.
5608	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5609	* out-of-sync. Make sure to update the required fields
5610	* before using them.
5611	*
5612	* @remarks No-long-jump zone!!!
5613	*/
5614	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5615	{
5616	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5617	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5618	return VINF_SUCCESS;
5619	}
5620
5621
5622	/**
5623	* Saves the entire guest state from the currently active VMCS into the
5624	* guest-CPU context. This essentially VMREADs all guest-data.
5625	*
5626	* @returns VBox status code.
5627	* @param pVCpu Pointer to the VMCPU.
5628	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5629	* out-of-sync. Make sure to update the required fields
5630	* before using them.
5631	*/
5632	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5633	{
5634	Assert(pVCpu);
5635	Assert(pMixedCtx);
5636
5637	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5638	return VINF_SUCCESS;
5639
5640	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5641	there is no real need to. */
5642	if (VMMRZCallRing3IsEnabled(pVCpu))
5643	VMMR0LogFlushDisable(pVCpu);
5644	else
5645	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5646	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5647
5648	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5649	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5650
5651	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5652	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5653
5654	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5655	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5656
5657	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5658	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5659
5660	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5661	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5662
5663	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5664	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5665
5666	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5667	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5668
5669	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5670	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5671
5672	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5673	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5674
5675	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5676	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5677
5678	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5679	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5680
5681	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5682	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5683
5684	if (VMMRZCallRing3IsEnabled(pVCpu))
5685	VMMR0LogFlushEnable(pVCpu);
5686
5687	return rc;
5688	}
5689
5690
5691	/**
5692	* Check per-VM and per-VCPU force flag actions that require us to go back to
5693	* ring-3 for one reason or another.
5694	*
5695	* @returns VBox status code (information status code included).
5696	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5697	* ring-3.
5698	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5699	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5700	* interrupts)
5701	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5702	* all EMTs to be in ring-3.
5703	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5704	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5705	* to the EM loop.
5706	*
5707	* @param pVM Pointer to the VM.
5708	* @param pVCpu Pointer to the VMCPU.
5709	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5710	* out-of-sync. Make sure to update the required fields
5711	* before using them.
5712	*/
5713	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5714	{
5715	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5716
5717	int rc = VERR_INTERNAL_ERROR_5;
5718	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5719	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5720	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5721	{
5722	/* We need the control registers now, make sure the guest-CPU context is updated. */
5723	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5724	AssertRCReturn(rc, rc);
5725
5726	/* Pending HM CR3 sync. */
5727	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5728	{
5729	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5730	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5731	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5732	}
5733
5734	/* Pending HM PAE PDPEs. */
5735	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5736	{
5737	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5738	AssertRC(rc);
5739	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5740	}
5741
5742	/* Pending PGM C3 sync. */
5743	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5744	{
5745	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5746	if (rc != VINF_SUCCESS)
5747	{
5748	AssertRC(rc);
5749	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5750	return rc;
5751	}
5752	}
5753
5754	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5755	/* -XXX- what was that about single stepping? */
5756	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5757	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5758	{
5759	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5760	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5761	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5762	return rc;
5763	}
5764
5765	/* Pending VM request packets, such as hardware interrupts. */
5766	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5767	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5768	{
5769	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5770	return VINF_EM_PENDING_REQUEST;
5771	}
5772
5773	/* Pending PGM pool flushes. */
5774	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5775	{
5776	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5777	return VINF_PGM_POOL_FLUSH_PENDING;
5778	}
5779
5780	/* Pending DMA requests. */
5781	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5782	{
5783	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5784	return VINF_EM_RAW_TO_R3;
5785	}
5786	}
5787
5788	/* Paranoia. */
5789	Assert(rc != VERR_EM_INTERPRETER);
5790	return VINF_SUCCESS;
5791	}
5792
5793
5794	/**
5795	* Converts any TRPM trap into a pending HM event. This is typically used when
5796	* entering from ring-3 (not longjmp returns).
5797	*
5798	* @param pVCpu Pointer to the VMCPU.
5799	*/
5800	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5801	{
5802	Assert(TRPMHasTrap(pVCpu));
5803	Assert(!pVCpu->hm.s.Event.fPending);
5804
5805	uint8_t uVector;
5806	TRPMEVENT enmTrpmEvent;
5807	RTGCUINT uErrCode;
5808	RTGCUINTPTR GCPtrFaultAddress;
5809	uint8_t cbInstr;
5810
5811	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5812	AssertRC(rc);
5813
5814	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5815	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5816	if (enmTrpmEvent == TRPM_TRAP)
5817	{
5818	switch (uVector)
5819	{
5820	case X86_XCPT_BP:
5821	case X86_XCPT_OF:
5822	{
5823	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5824	break;
5825	}
5826
5827	case X86_XCPT_PF:
5828	case X86_XCPT_DF:
5829	case X86_XCPT_TS:
5830	case X86_XCPT_NP:
5831	case X86_XCPT_SS:
5832	case X86_XCPT_GP:
5833	case X86_XCPT_AC:
5834	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5835	/* no break! */
5836	default:
5837	{
5838	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5839	break;
5840	}
5841	}
5842	}
5843	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5844	{
5845	if (uVector == X86_XCPT_NMI)
5846	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5847	else
5848	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5849	}
5850	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5851	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5852	else
5853	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5854
5855	rc = TRPMResetTrap(pVCpu);
5856	AssertRC(rc);
5857	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5858	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5859
5860	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5861	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
5862	}
5863
5864
5865	/**
5866	* Converts any pending HM event into a TRPM trap. Typically used when leaving
5867	* VT-x to execute any instruction.
5868	*
5869	* @param pvCpu Pointer to the VMCPU.
5870	*/
5871	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
5872	{
5873	Assert(pVCpu->hm.s.Event.fPending);
5874
5875	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5876	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5877	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5878	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5879
5880	/* If a trap was already pending, we did something wrong! */
5881	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5882
5883	TRPMEVENT enmTrapType;
5884	switch (uVectorType)
5885	{
5886	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5887	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5888	enmTrapType = TRPM_HARDWARE_INT;
5889	break;
5890	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5891	enmTrapType = TRPM_SOFTWARE_INT;
5892	break;
5893	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5894	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5895	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5896	enmTrapType = TRPM_TRAP;
5897	break;
5898	default:
5899	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5900	enmTrapType = TRPM_32BIT_HACK;
5901	break;
5902	}
5903
5904	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5905
5906	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5907	AssertRC(rc);
5908
5909	if (fErrorCodeValid)
5910	TRPMSetErrorCode(pVCpu, uErrorCode);
5911
5912	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5913	&& uVector == X86_XCPT_PF)
5914	{
5915	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5916	}
5917	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5918	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5919	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5920	{
5921	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5922	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5923	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5924	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5925	}
5926	pVCpu->hm.s.Event.fPending = false;
5927	}
5928
5929
5930	/**
5931	* Does the necessary state syncing before doing a longjmp to ring-3.
5932	*
5933	* @param pVM Pointer to the VM.
5934	* @param pVCpu Pointer to the VMCPU.
5935	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5936	* out-of-sync. Make sure to update the required fields
5937	* before using them.
5938	* @param rcExit The reason for exiting to ring-3. Can be
5939	* VINF_VMM_UNKNOWN_RING3_CALL.
5940	*
5941	* @remarks No-long-jmp zone!!!
5942	*/
5943	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5944	{
5945	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5946	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5947
5948	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5949	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5950	AssertRC(rc);
5951
5952	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
5953	if (CPUMIsGuestFPUStateActive(pVCpu))
5954	{
5955	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5956	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5957	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5958	}
5959
5960	/* Restore host debug registers if necessary and resync on next R0 reentry. */
5961	if (CPUMIsGuestDebugStateActive(pVCpu))
5962	{
5963	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5964	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5965	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5966	}
5967	else if (CPUMIsHyperDebugStateActive(pVCpu))
5968	{
5969	CPUMR0LoadHostDebugState(pVM, pVCpu);
5970	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5971	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5972	}
5973
5974	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
5975	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
5976	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
5977	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
5978	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
5979	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
5980	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
5981	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5982
5983	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5984	}
5985
5986
5987	/**
5988	* An action requires us to go back to ring-3. This function does the necessary
5989	* steps before we can safely return to ring-3. This is not the same as longjmps
5990	* to ring-3, this is voluntary.
5991	*
5992	* @param pVM Pointer to the VM.
5993	* @param pVCpu Pointer to the VMCPU.
5994	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5995	* out-of-sync. Make sure to update the required fields
5996	* before using them.
5997	* @param rcExit The reason for exiting to ring-3. Can be
5998	* VINF_VMM_UNKNOWN_RING3_CALL.
5999	*/
6000	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6001	{
6002	Assert(pVM);
6003	Assert(pVCpu);
6004	Assert(pMixedCtx);
6005	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6006
6007	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6008	{
6009	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
6010	return;
6011	}
6012	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6013	{
6014	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6015	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6016	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6017	pVCpu->hm.s.vmx.LastError.idCurrentCpu = RTMpCpuId();
6018	return;
6019	}
6020
6021	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6022	VMMRZCallRing3Disable(pVCpu);
6023	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6024
6025	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6026	if (pVCpu->hm.s.Event.fPending)
6027	{
6028	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6029	Assert(!pVCpu->hm.s.Event.fPending);
6030	}
6031
6032	/* Sync. the guest state. */
6033	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
6034	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6035
6036	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6037	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6038	\| CPUM_CHANGED_LDTR
6039	\| CPUM_CHANGED_GDTR
6040	\| CPUM_CHANGED_IDTR
6041	\| CPUM_CHANGED_TR
6042	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6043
6044	/* On our way back from ring-3 the following needs to be done. */
6045	/** @todo This can change with preemption hooks. */
6046	if (rcExit == VINF_EM_RAW_INTERRUPT)
6047	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
6048	else
6049	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
6050
6051	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6052	VMMRZCallRing3Enable(pVCpu);
6053	}
6054
6055
6056	/**
6057	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6058	* longjump to ring-3 and possibly get preempted.
6059	*
6060	* @param pVCpu Pointer to the VMCPU.
6061	* @param enmOperation The operation causing the ring-3 longjump.
6062	* @param pvUser The user argument (pointer to the possibly
6063	* out-of-date guest-CPU context).
6064	*
6065	* @remarks Must never be called with @a enmOperation ==
6066	* VMMCALLRING3_VM_R0_ASSERTION.
6067	*/
6068	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6069	{
6070	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6071	Assert(pVCpu);
6072	Assert(pvUser);
6073	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6074	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6075
6076	VMMRZCallRing3Disable(pVCpu);
6077	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6078	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6079	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
6080	VMMRZCallRing3Enable(pVCpu);
6081	}
6082
6083
6084	/**
6085	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6086	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6087	*
6088	* @param pVCpu Pointer to the VMCPU.
6089	*/
6090	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6091	{
6092	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6093	{
6094	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6095	{
6096	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6097	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6098	AssertRC(rc);
6099	}
6100	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6101	}
6102
6103
6104	/**
6105	* Injects any pending events into the guest if the guest is in a state to
6106	* receive them.
6107	*
6108	* @returns VBox status code (informational status codes included).
6109	* @param pVCpu Pointer to the VMCPU.
6110	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6111	* out-of-sync. Make sure to update the required fields
6112	* before using them.
6113	*/
6114	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6115	{
6116	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6117	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6118	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6119	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6120
6121	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6122	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6123	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6124	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6125	Assert(!TRPMHasTrap(pVCpu));
6126
6127	int rc = VINF_SUCCESS;
6128	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6129	{
6130	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6131	bool fInject = true;
6132	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6133	{
6134	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6135	AssertRCReturn(rc, rc);
6136	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6137	if ( fBlockInt
6138	\|\| fBlockSti
6139	\|\| fBlockMovSS)
6140	{
6141	fInject = false;
6142	}
6143	}
6144	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6145	&& ( fBlockMovSS
6146	\|\| fBlockSti))
6147	{
6148	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6149	fInject = false;
6150	}
6151
6152	if (fInject)
6153	{
6154	Log4(("Injecting pending event vcpu[%RU32]\n", pVCpu->idCpu));
6155	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6156	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6157	AssertRCReturn(rc, rc);
6158	pVCpu->hm.s.Event.fPending = false;
6159
6160	#ifdef VBOX_WITH_STATISTICS
6161	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6162	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6163	else
6164	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6165	#endif
6166	}
6167	else
6168	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6169	} /** @todo SMI. SMIs take priority over NMIs. */
6170	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6171	{
6172	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6173	if ( !fBlockMovSS
6174	&& !fBlockSti)
6175	{
6176	Log4(("Injecting NMI\n"));
6177	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6178	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6179	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6180	0 /* GCPtrFaultAddress */, &uIntrState);
6181	AssertRCReturn(rc, rc);
6182	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6183
6184	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6185	}
6186	else
6187	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6188	}
6189	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
6190	{
6191	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6192	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6193	AssertRCReturn(rc, rc);
6194	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6195	if ( !fBlockInt
6196	&& !fBlockSti
6197	&& !fBlockMovSS)
6198	{
6199	uint8_t u8Interrupt;
6200	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6201	if (RT_SUCCESS(rc))
6202	{
6203	Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6204	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6205	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6206	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6207	0 /* GCPtrFaultAddress */, &uIntrState);
6208
6209	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6210	}
6211	else
6212	{
6213	/** @todo Does this actually happen? If not turn it into an assertion. */
6214	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6215	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6216	rc = VINF_SUCCESS;
6217	}
6218	}
6219	else
6220	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6221	}
6222
6223	/*
6224	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6225	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6226	*/
6227	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6228	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6229	int rc2 = VINF_SUCCESS;
6230	if ( fBlockSti
6231	\|\| fBlockMovSS)
6232	{
6233	if (!DBGFIsStepping(pVCpu))
6234	{
6235	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6236	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6237	{
6238	/*
6239	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6240	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6241	*/
6242	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6243	AssertRCReturn(rc, rc);
6244	}
6245	}
6246	else
6247	{
6248	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6249	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6250	uIntrState = 0;
6251	}
6252	}
6253
6254	/*
6255	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6256	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6257	*/
6258	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6259	AssertRC(rc2);
6260
6261	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6262	return rc;
6263	}
6264
6265
6266	/**
6267	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6268	*
6269	* @param pVCpu Pointer to the VMCPU.
6270	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6271	* out-of-sync. Make sure to update the required fields
6272	* before using them.
6273	*/
6274	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6275	{
6276	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6277	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6278	}
6279
6280
6281	/**
6282	* Injects a double-fault (#DF) exception into the VM.
6283	*
6284	* @returns VBox status code (informational status code included).
6285	* @param pVCpu Pointer to the VMCPU.
6286	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6287	* out-of-sync. Make sure to update the required fields
6288	* before using them.
6289	*/
6290	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6291	{
6292	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6293	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6294	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6295	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6296	puIntrState);
6297	}
6298
6299
6300	/**
6301	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6302	*
6303	* @param pVCpu Pointer to the VMCPU.
6304	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6305	* out-of-sync. Make sure to update the required fields
6306	* before using them.
6307	*/
6308	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6309	{
6310	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6311	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6312	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6313	}
6314
6315
6316	/**
6317	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6318	*
6319	* @param pVCpu Pointer to the VMCPU.
6320	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6321	* out-of-sync. Make sure to update the required fields
6322	* before using them.
6323	* @param cbInstr The value of RIP that is to be pushed on the guest
6324	* stack.
6325	*/
6326	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6327	{
6328	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6329	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6330	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6331	}
6332
6333
6334	/**
6335	* Injects a general-protection (#GP) fault into the VM.
6336	*
6337	* @returns VBox status code (informational status code included).
6338	* @param pVCpu Pointer to the VMCPU.
6339	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6340	* out-of-sync. Make sure to update the required fields
6341	* before using them.
6342	* @param u32ErrorCode The error code associated with the #GP.
6343	*/
6344	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6345	uint32_t *puIntrState)
6346	{
6347	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6348	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6349	if (fErrorCodeValid)
6350	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6351	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6352	puIntrState);
6353	}
6354
6355
6356	/**
6357	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6358	*
6359	* @param pVCpu Pointer to the VMCPU.
6360	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6361	* out-of-sync. Make sure to update the required fields
6362	* before using them.
6363	* @param uVector The software interrupt vector number.
6364	* @param cbInstr The value of RIP that is to be pushed on the guest
6365	* stack.
6366	*/
6367	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6368	{
6369	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6370	if ( uVector == X86_XCPT_BP
6371	\|\| uVector == X86_XCPT_OF)
6372	{
6373	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6374	}
6375	else
6376	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6377	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6378	}
6379
6380
6381	/**
6382	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6383	* stack.
6384	*
6385	* @returns VBox status code (information status code included).
6386	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6387	* @param pVM Pointer to the VM.
6388	* @param pMixedCtx Pointer to the guest-CPU context.
6389	* @param uValue The value to push to the guest stack.
6390	*/
6391	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6392	{
6393	/*
6394	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6395	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6396	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6397	*/
6398	if (pMixedCtx->sp == 1)
6399	return VINF_EM_RESET;
6400	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6401	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6402	AssertRCReturn(rc, rc);
6403	return rc;
6404	}
6405
6406
6407	/**
6408	* Injects an event into the guest upon VM-entry by updating the relevant fields
6409	* in the VM-entry area in the VMCS.
6410	*
6411	* @returns VBox status code (informational error codes included).
6412	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6413	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6414	*
6415	* @param pVCpu Pointer to the VMCPU.
6416	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6417	* be out-of-sync. Make sure to update the required
6418	* fields before using them.
6419	* @param u64IntrInfo The VM-entry interruption-information field.
6420	* @param cbInstr The VM-entry instruction length in bytes (for
6421	* software interrupts, exceptions and privileged
6422	* software exceptions).
6423	* @param u32ErrCode The VM-entry exception error code.
6424	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6425	* @param puIntrState Pointer to the current guest interruptibility-state.
6426	* This interruptibility-state will be updated if
6427	* necessary. This cannot not be NULL.
6428	*
6429	* @remarks No-long-jump zone!!!
6430	* @remarks Requires CR0!
6431	*/
6432	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6433	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6434	{
6435	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6436	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6437	Assert(puIntrState);
6438	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6439
6440	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6441	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6442
6443	#ifdef VBOX_STRICT
6444	/* Validate the error-code-valid bit for hardware exceptions. */
6445	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6446	{
6447	switch (uVector)
6448	{
6449	case X86_XCPT_PF:
6450	case X86_XCPT_DF:
6451	case X86_XCPT_TS:
6452	case X86_XCPT_NP:
6453	case X86_XCPT_SS:
6454	case X86_XCPT_GP:
6455	case X86_XCPT_AC:
6456	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6457	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6458	/* fallthru */
6459	default:
6460	break;
6461	}
6462	}
6463	#endif
6464
6465	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6466	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6467	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6468
6469	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6470
6471	/* We require CR0 to check if the guest is in real-mode. */
6472	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6473	AssertRCReturn(rc, rc);
6474
6475	/*
6476	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6477	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6478	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6479	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6480	*/
6481	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6482	{
6483	PVM pVM = pVCpu->CTX_SUFF(pVM);
6484	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6485	{
6486	Assert(PDMVmmDevHeapIsEnabled(pVM));
6487	Assert(pVM->hm.s.vmx.pRealModeTSS);
6488
6489	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6490	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6491	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6492	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6493	AssertRCReturn(rc, rc);
6494	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6495
6496	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6497	const size_t cbIdtEntry = 4;
6498	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6499	{
6500	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6501	if (uVector == X86_XCPT_DF)
6502	return VINF_EM_RESET;
6503	else if (uVector == X86_XCPT_GP)
6504	{
6505	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6506	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6507	}
6508
6509	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6510	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6511	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6512	}
6513
6514	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6515	uint16_t uGuestIp = pMixedCtx->ip;
6516	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6517	{
6518	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6519	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6520	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6521	}
6522	else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6523	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6524
6525	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6526	uint16_t offIdtEntry = 0;
6527	RTSEL selIdtEntry = 0;
6528	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6529	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6530	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6531	AssertRCReturn(rc, rc);
6532
6533	/* Construct the stack frame for the interrupt/exception handler. */
6534	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6535	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6536	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6537	AssertRCReturn(rc, rc);
6538
6539	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6540	if (rc == VINF_SUCCESS)
6541	{
6542	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6543	pMixedCtx->rip = offIdtEntry;
6544	pMixedCtx->cs.Sel = selIdtEntry;
6545	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6546	if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6547	&& uVector == X86_XCPT_PF)
6548	{
6549	pMixedCtx->cr2 = GCPtrFaultAddress;
6550	}
6551	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6552	\| HM_CHANGED_GUEST_RIP
6553	\| HM_CHANGED_GUEST_RFLAGS
6554	\| HM_CHANGED_GUEST_RSP;
6555
6556	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6557	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6558	{
6559	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6560	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6561	Log4(("Clearing inhibition due to STI.\n"));
6562	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6563	}
6564	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6565	}
6566	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6567	return rc;
6568	}
6569	else
6570	{
6571	/*
6572	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6573	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6574	*/
6575	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6576	}
6577	}
6578
6579	/* Validate. */
6580	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6581	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6582	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6583
6584	/* Inject. */
6585	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6586	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6587	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6588	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6589
6590	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6591	&& uVector == X86_XCPT_PF)
6592	{
6593	pMixedCtx->cr2 = GCPtrFaultAddress;
6594	}
6595
6596	Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6597	u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6598
6599	AssertRCReturn(rc, rc);
6600	return rc;
6601	}
6602
6603
6604	/**
6605	* Enters the VT-x session.
6606	*
6607	* @returns VBox status code.
6608	* @param pVM Pointer to the VM.
6609	* @param pVCpu Pointer to the VMCPU.
6610	* @param pCpu Pointer to the CPU info struct.
6611	*/
6612	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6613	{
6614	AssertPtr(pVM);
6615	AssertPtr(pVCpu);
6616	Assert(pVM->hm.s.vmx.fSupported);
6617	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6618	NOREF(pCpu);
6619
6620	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6621
6622	#ifdef VBOX_STRICT
6623	/* Make sure we're in VMX root mode. */
6624	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6625	if (!(u32HostCR4 & X86_CR4_VMXE))
6626	{
6627	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6628	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6629	}
6630	#endif
6631
6632	/* Load the active VMCS as the current one. */
6633	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6634	if (RT_FAILURE(rc))
6635	return rc;
6636
6637	/** @todo this will change with preemption hooks where can VMRESUME as long
6638	* as we're no preempted. */
6639	pVCpu->hm.s.fResumeVM = false;
6640	return VINF_SUCCESS;
6641	}
6642
6643
6644	/**
6645	* Leaves the VT-x session.
6646	*
6647	* @returns VBox status code.
6648	* @param pVM Pointer to the VM.
6649	* @param pVCpu Pointer to the VMCPU.
6650	* @param pCtx Pointer to the guest-CPU context.
6651	*/
6652	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6653	{
6654	AssertPtr(pVCpu);
6655	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6656	NOREF(pVM);
6657	NOREF(pCtx);
6658
6659	/** @todo this will change with preemption hooks where we only VMCLEAR when
6660	* we are actually going to be preempted, not all the time like we
6661	* currently do. */
6662
6663	/* Restore host-state bits that VT-x only restores partially. */
6664	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6665	{
6666	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6667	/** @todo r=ramshankar: This is broken when
6668	* VBOX_WITH_VMMR0_DISABLE_PREEMPTION is not defined. As
6669	* VMXRestoreHostState() may unconditionally enables interrupts. */
6670	#error "VMM: Fix Me! Make VMXRestoreHostState() function to skip cli/sti."
6671	#else
6672	Assert(ASMIntAreEnabled());
6673	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6674	#endif
6675	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6676	}
6677
6678	/*
6679	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6680	* and mark the VMCS launch-state as "clear".
6681	*/
6682	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6683	return rc;
6684	}
6685
6686
6687	/**
6688	* Saves the host state in the VMCS host-state.
6689	* Sets up the VM-exit MSR-load area.
6690	*
6691	* The CPU state will be loaded from these fields on every successful VM-exit.
6692	*
6693	* @returns VBox status code.
6694	* @param pVM Pointer to the VM.
6695	* @param pVCpu Pointer to the VMCPU.
6696	*
6697	* @remarks No-long-jump zone!!!
6698	*/
6699	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6700	{
6701	AssertPtr(pVM);
6702	AssertPtr(pVCpu);
6703	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6704
6705	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6706
6707	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6708	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6709	return VINF_SUCCESS;
6710
6711	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6712	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6713
6714	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6715	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6716
6717	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6718	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6719
6720	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6721	return rc;
6722	}
6723
6724
6725	/**
6726	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6727	* loaded from these fields on every successful VM-entry.
6728	*
6729	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6730	* Sets up the VM-entry controls.
6731	* Sets up the appropriate VMX non-root function to execute guest code based on
6732	* the guest CPU mode.
6733	*
6734	* @returns VBox status code.
6735	* @param pVM Pointer to the VM.
6736	* @param pVCpu Pointer to the VMCPU.
6737	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6738	* out-of-sync. Make sure to update the required fields
6739	* before using them.
6740	*
6741	* @remarks No-long-jump zone!!!
6742	*/
6743	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6744	{
6745	AssertPtr(pVM);
6746	AssertPtr(pVCpu);
6747	AssertPtr(pMixedCtx);
6748	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6749
6750	#ifdef LOG_ENABLED
6751	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
6752	* probably not initialized yet? Anyway this will do for now. */
6753	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
6754	VMMR0LogFlushDisable(pVCpu);
6755	#endif
6756
6757	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6758
6759	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6760
6761	/* Determine real-on-v86 mode. */
6762	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6763	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6764	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6765	{
6766	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6767	}
6768
6769	/*
6770	* Load the guest-state into the VMCS.
6771	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6772	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6773	*/
6774	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6775	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6776
6777	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6778	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6779
6780	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6781	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6782
6783	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6784	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6785
6786	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6787	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6788	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6789
6790	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6791	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6792
6793	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6794	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6795
6796	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6797	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6798
6799	/* Must be done after hmR0VmxLoadGuestDebugRegs() as it may update eflags.TF for debugging purposes. */
6800	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6801	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6802
6803	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6804	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6805
6806	/* Clear any unused and reserved bits. */
6807	pVCpu->hm.s.fContextUseFlags &= ~( HM_CHANGED_GUEST_CR2
6808	\| HM_CHANGED_GUEST_MSR /* legacy */);
6809
6810	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6811	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p idCpu=%RU32 fContextUseFlags=%#RX32\n",
6812	pVM, pVCpu, pVCpu->idCpu, pVCpu->hm.s.fContextUseFlags));
6813
6814	#ifdef LOG_ENABLED
6815	/* Only reenable log-flushing if the caller has it enabled. */
6816	if (!fCallerDisabledLogFlush)
6817	VMMR0LogFlushEnable(pVCpu);
6818	#endif
6819
6820	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6821	return rc;
6822	}
6823
6824
6825	/**
6826	* Loads the guest state into the VMCS guest-state area.
6827	*
6828	* @returns VBox status code.
6829	* @param pVM Pointer to the VM.
6830	* @param pVCpu Pointer to the VMCPU.
6831	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6832	* out-of-sync. Make sure to update the required fields
6833	* before using them.
6834	*
6835	* @remarks No-long-jump zone!!!
6836	*/
6837	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6838	{
6839	/*
6840	* Avoid reloading the guest state on longjmp reentrants and do it lazily just before executing the guest.
6841	* This only helps when we get rescheduled more than once to a different host CPU on a longjmp trip before
6842	* finally executing guest code.
6843	*/
6844	return VINF_SUCCESS;
6845	}
6846
6847
6848	/**
6849	* Does the preparations before executing guest code in VT-x.
6850	*
6851	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6852	* recompiler. We must be cautious what we do here regarding committing
6853	* guest-state information into the VMCS assuming we assuredly execute the
6854	* guest in VT-x. If we fall back to the recompiler after updating the VMCS and
6855	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6856	* that the recompiler can (and should) use them when it resumes guest
6857	* execution. Otherwise such operations must be done when we can no longer
6858	* exit to ring-3.
6859	*
6860	* @returns VBox status code (informational status codes included).
6861	* @retval VINF_SUCCESS if we can proceed with running the guest.
6862	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6863	* into the guest.
6864	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6865	*
6866	* @param pVM Pointer to the VM.
6867	* @param pVCpu Pointer to the VMCPU.
6868	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6869	* out-of-sync. Make sure to update the required fields
6870	* before using them.
6871	* @param pVmxTransient Pointer to the VMX transient structure.
6872	*
6873	* @remarks Called with preemption disabled.
6874	*/
6875	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6876	{
6877	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6878
6879	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6880	PGMRZDynMapFlushAutoSet(pVCpu);
6881	#endif
6882
6883	/* Check force flag actions that might require us to go back to ring-3. */
6884	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6885	if (rc != VINF_SUCCESS)
6886	return rc;
6887
6888	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6889	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6890	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6891	{
6892	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6893	RTGCPHYS GCPhysApicBase;
6894	GCPhysApicBase = pMixedCtx->msrApicBase;
6895	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6896
6897	/* Unalias any existing mapping. */
6898	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6899	AssertRCReturn(rc, rc);
6900
6901	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6902	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6903	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6904	AssertRCReturn(rc, rc);
6905
6906	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6907	}
6908
6909	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6910	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6911	pVmxTransient->uEFlags = ASMIntDisableFlags();
6912	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6913	{
6914	ASMSetFlags(pVmxTransient->uEFlags);
6915	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6916	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6917	return VINF_EM_RAW_INTERRUPT;
6918	}
6919	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6920	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6921	#endif
6922
6923	/*
6924	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6925	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6926	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6927	*/
6928	/** @todo Rework event evaluation and injection to be completely separate. */
6929	if (TRPMHasTrap(pVCpu))
6930	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
6931
6932	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6933	AssertRCReturn(rc, rc);
6934	return rc;
6935	}
6936
6937
6938	/**
6939	* Prepares to run guest code in VT-x and we've committed to doing so. This
6940	* means there is no backing out to ring-3 or anywhere else at this
6941	* point.
6942	*
6943	* @param pVM Pointer to the VM.
6944	* @param pVCpu Pointer to the VMCPU.
6945	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6946	* out-of-sync. Make sure to update the required fields
6947	* before using them.
6948	* @param pVmxTransient Pointer to the VMX transient structure.
6949	*
6950	* @remarks Called with preemption disabled.
6951	* @remarks No-long-jump zone!!!
6952	*/
6953	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6954	{
6955	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6956	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6957
6958	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6959	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6960	pVmxTransient->uEFlags = ASMIntDisableFlags();
6961	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6962	#endif
6963
6964	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6965	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6966	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6967	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6968	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6969	#endif
6970	int rc = VINF_SUCCESS;
6971	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6972	{
6973	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6974	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6975	}
6976	else if (pVCpu->hm.s.fContextUseFlags)
6977	{
6978	rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
6979	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6980	}
6981	AssertRC(rc);
6982	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6983
6984	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6985	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6986	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6987
6988	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6989	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6990	{
6991	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6992	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6993	}
6994
6995	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6996	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6997	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6998
6999	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7000
7001	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7002	to start executing. */
7003
7004	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7005	/*
7006	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7007	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7008	*/
7009	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7010	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7011	{
7012	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7013	uint64_t u64HostTscAux = 0;
7014	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7015	AssertRC(rc2);
7016	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7017	}
7018	#endif
7019	}
7020
7021
7022	/**
7023	* Performs some essential restoration of state after running guest code in
7024	* VT-x.
7025	*
7026	* @param pVM Pointer to the VM.
7027	* @param pVCpu Pointer to the VMCPU.
7028	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7029	* out-of-sync. Make sure to update the required fields
7030	* before using them.
7031	* @param pVmxTransient Pointer to the VMX transient structure.
7032	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7033	*
7034	* @remarks Called with interrupts disabled.
7035	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7036	* unconditionally when it is safe to do so.
7037	*/
7038	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7039	{
7040	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7041
7042	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7043	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7044	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7045	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7046	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7047
7048	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7049	{
7050	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7051	/* Restore host's TSC_AUX. */
7052	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7053	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7054	#endif
7055	/** @todo Find a way to fix hardcoding a guestimate. */
7056	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7057	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7058	}
7059
7060	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7061	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7062	Assert(!(ASMGetFlags() & X86_EFL_IF));
7063	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7064
7065	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
7066	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7067
7068	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7069	uint32_t uExitReason;
7070	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7071	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7072	AssertRC(rc);
7073	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7074	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7075
7076	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
7077	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7078
7079	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7080	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7081	{
7082	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7083	pVmxTransient->fVMEntryFailed));
7084	return;
7085	}
7086
7087	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7088	{
7089	/* Update the guest interruptibility-state from the VMCS. */
7090	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7091	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7092	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7093	AssertRC(rc);
7094	#endif
7095	/*
7096	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7097	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7098	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7099	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7100	*/
7101	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7102	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7103	{
7104	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7105	AssertRC(rc);
7106	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7107	}
7108	}
7109	}
7110
7111
7112	/**
7113	* Runs the guest code using VT-x.
7114	*
7115	* @returns VBox status code.
7116	* @param pVM Pointer to the VM.
7117	* @param pVCpu Pointer to the VMCPU.
7118	* @param pCtx Pointer to the guest-CPU context.
7119	*
7120	* @remarks Called with preemption disabled.
7121	*/
7122	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7123	{
7124	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7125	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7126
7127	VMXTRANSIENT VmxTransient;
7128	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7129	int rc = VERR_INTERNAL_ERROR_5;
7130	uint32_t cLoops = 0;
7131
7132	for (;; cLoops++)
7133	{
7134	Assert(!HMR0SuspendPending());
7135	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7136	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7137	(unsigned)RTMpCpuId(), cLoops));
7138
7139	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7140	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7141	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7142	if (rc != VINF_SUCCESS)
7143	break;
7144
7145	/*
7146	* No longjmps to ring-3 from this point on!!!
7147	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7148	* This also disables flushing of the R0-logger instance (if any).
7149	*/
7150	VMMRZCallRing3Disable(pVCpu);
7151	VMMRZCallRing3RemoveNotification(pVCpu);
7152	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7153
7154	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7155	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7156
7157	/*
7158	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7159	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7160	*/
7161	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7162	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7163	{
7164	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7165	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7166	return rc;
7167	}
7168
7169	/* Handle the VM-exit. */
7170	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7171	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7172	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7173	HMVMX_START_EXIT_DISPATCH_PROF();
7174	#ifdef HMVMX_USE_FUNCTION_TABLE
7175	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7176	#else
7177	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7178	#endif
7179	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7180	if (rc != VINF_SUCCESS)
7181	break;
7182	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7183	{
7184	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7185	rc = VINF_EM_RAW_INTERRUPT;
7186	break;
7187	}
7188	}
7189
7190	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7191	if (rc == VERR_EM_INTERPRETER)
7192	rc = VINF_EM_RAW_EMULATE_INSTR;
7193	else if (rc == VINF_EM_RESET)
7194	rc = VINF_EM_TRIPLE_FAULT;
7195	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7196	return rc;
7197	}
7198
7199
7200	#ifndef HMVMX_USE_FUNCTION_TABLE
7201	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7202	{
7203	int rc;
7204	switch (rcReason)
7205	{
7206	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7207	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7208	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7209	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7210	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7211	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7212	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7213	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7214	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7215	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7216	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7217	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7218	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7219	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7220	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7221	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7222	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7223	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7224	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7225	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7226	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7227	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7228	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7229	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7230	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7231	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7232	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7233	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7234	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7235	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7236	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7237	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7238	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7239
7240	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7241	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7242	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7243	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7244	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7245	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7246	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7247	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7248	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7249
7250	case VMX_EXIT_VMCALL:
7251	case VMX_EXIT_VMCLEAR:
7252	case VMX_EXIT_VMLAUNCH:
7253	case VMX_EXIT_VMPTRLD:
7254	case VMX_EXIT_VMPTRST:
7255	case VMX_EXIT_VMREAD:
7256	case VMX_EXIT_VMRESUME:
7257	case VMX_EXIT_VMWRITE:
7258	case VMX_EXIT_VMXOFF:
7259	case VMX_EXIT_VMXON:
7260	case VMX_EXIT_INVEPT:
7261	case VMX_EXIT_INVVPID:
7262	case VMX_EXIT_VMFUNC:
7263	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7264	break;
7265	default:
7266	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7267	break;
7268	}
7269	return rc;
7270	}
7271	#endif
7272
7273	#ifdef DEBUG
7274	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7275	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7276	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7277
7278	# define HMVMX_ASSERT_PREEMPT_CPUID() \
7279	do \
7280	{ \
7281	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7282	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7283	} while (0)
7284
7285	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7286	do { \
7287	AssertPtr(pVCpu); \
7288	AssertPtr(pMixedCtx); \
7289	AssertPtr(pVmxTransient); \
7290	Assert(pVmxTransient->fVMEntryFailed == false); \
7291	Assert(ASMIntAreEnabled()); \
7292	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7293	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7294	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7295	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7296	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7297	HMVMX_ASSERT_PREEMPT_CPUID(); \
7298	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7299	} while (0)
7300
7301	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7302	do { \
7303	Log4Func(("\n")); \
7304	} while(0)
7305	#else /* Release builds */
7306	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7307	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7308	#endif
7309
7310
7311	/**
7312	* Advances the guest RIP after reading it from the VMCS.
7313	*
7314	* @returns VBox status code.
7315	* @param pVCpu Pointer to the VMCPU.
7316	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7317	* out-of-sync. Make sure to update the required fields
7318	* before using them.
7319	* @param pVmxTransient Pointer to the VMX transient structure.
7320	*
7321	* @remarks No-long-jump zone!!!
7322	*/
7323	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7324	{
7325	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7326	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7327	AssertRCReturn(rc, rc);
7328
7329	pMixedCtx->rip += pVmxTransient->cbInstr;
7330	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7331	return rc;
7332	}
7333
7334
7335	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7336	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7337	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7338
7339	/** @name VM-exit handlers.
7340	* @{
7341	*/
7342
7343	/**
7344	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7345	*/
7346	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7347	{
7348	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7349	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7350	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
7351	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
7352	Assert(ASMIntAreEnabled());
7353	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
7354	return VINF_SUCCESS;
7355	#endif
7356	return VINF_EM_RAW_INTERRUPT;
7357	}
7358
7359
7360	/**
7361	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
7362	*/
7363	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7364	{
7365	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7366	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7367
7368	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7369	AssertRCReturn(rc, rc);
7370
7371	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7372	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7373	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7374	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
7375
7376	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7377	{
7378	/*
7379	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
7380	* anything we inject is not going to cause a VM-exit directly for the event being injected.
7381	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
7382	*
7383	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
7384	*/
7385	VMXDispatchHostNmi();
7386	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmi);
7387	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7388	return VINF_SUCCESS;
7389	}
7390
7391	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7392	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7393	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
7394	{
7395	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7396	return VINF_SUCCESS;
7397	}
7398	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7399	{
7400	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7401	return rc;
7402	}
7403
7404	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7405	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7406	switch (uIntrType)
7407	{
7408	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7409	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7410	/* no break */
7411	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7412	{
7413	switch (uVector)
7414	{
7415	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7416	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7417	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7418	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7419	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7420	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7421	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7422	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7423	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7424	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7425	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7426	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7427	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7428	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7429	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7430	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7431	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7432	#endif
7433	default:
7434	{
7435	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7436	AssertRCReturn(rc, rc);
7437
7438	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7439	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7440	{
7441	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7442	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7443	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7444	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7445	AssertRCReturn(rc, rc);
7446	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7447	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7448	0 /* GCPtrFaultAddress */);
7449	AssertRCReturn(rc, rc);
7450	}
7451	else
7452	{
7453	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7454	pVCpu->hm.s.u32HMError = uVector;
7455	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7456	}
7457	break;
7458	}
7459	}
7460	break;
7461	}
7462
7463	default:
7464	{
7465	pVCpu->hm.s.u32HMError = uExitIntrInfo;
7466	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7467	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7468	break;
7469	}
7470	}
7471	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7472	return rc;
7473	}
7474
7475
7476	/**
7477	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7478	*/
7479	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7480	{
7481	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7482
7483	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7484	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7485	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7486	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7487	AssertRCReturn(rc, rc);
7488
7489	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7490	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7491	return VINF_SUCCESS;
7492	}
7493
7494
7495	/**
7496	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7497	*/
7498	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7499	{
7500	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7501	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7502	pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
7503	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7504	}
7505
7506
7507	/**
7508	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7509	*/
7510	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7511	{
7512	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7513	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7514	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7515	}
7516
7517
7518	/**
7519	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7520	*/
7521	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7522	{
7523	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7524	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7525	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7526	}
7527
7528
7529	/**
7530	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7531	*/
7532	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7533	{
7534	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7535	PVM pVM = pVCpu->CTX_SUFF(pVM);
7536	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7537	if (RT_LIKELY(rc == VINF_SUCCESS))
7538	{
7539	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7540	Assert(pVmxTransient->cbInstr == 2);
7541	}
7542	else
7543	{
7544	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7545	rc = VERR_EM_INTERPRETER;
7546	}
7547	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7548	return rc;
7549	}
7550
7551
7552	/**
7553	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7554	*/
7555	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7556	{
7557	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7558	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7559	AssertRCReturn(rc, rc);
7560
7561	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7562	return VINF_EM_RAW_EMULATE_INSTR;
7563
7564	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7565	pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
7566	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7567	}
7568
7569
7570	/**
7571	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7572	*/
7573	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7574	{
7575	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7576	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7577	AssertRCReturn(rc, rc);
7578
7579	PVM pVM = pVCpu->CTX_SUFF(pVM);
7580	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7581	if (RT_LIKELY(rc == VINF_SUCCESS))
7582	{
7583	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7584	Assert(pVmxTransient->cbInstr == 2);
7585	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7586	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7587	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7588	}
7589	else
7590	{
7591	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7592	rc = VERR_EM_INTERPRETER;
7593	}
7594	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7595	return rc;
7596	}
7597
7598
7599	/**
7600	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7601	*/
7602	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7603	{
7604	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7605	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7606	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7607	AssertRCReturn(rc, rc);
7608
7609	PVM pVM = pVCpu->CTX_SUFF(pVM);
7610	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7611	if (RT_LIKELY(rc == VINF_SUCCESS))
7612	{
7613	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7614	Assert(pVmxTransient->cbInstr == 3);
7615	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7616	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7617	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7618	}
7619	else
7620	{
7621	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7622	rc = VERR_EM_INTERPRETER;
7623	}
7624	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7625	return rc;
7626	}
7627
7628
7629	/**
7630	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7631	*/
7632	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7633	{
7634	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7635	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7636	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7637	AssertRCReturn(rc, rc);
7638
7639	PVM pVM = pVCpu->CTX_SUFF(pVM);
7640	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7641	if (RT_LIKELY(rc == VINF_SUCCESS))
7642	{
7643	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7644	Assert(pVmxTransient->cbInstr == 2);
7645	}
7646	else
7647	{
7648	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7649	rc = VERR_EM_INTERPRETER;
7650	}
7651	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7652	return rc;
7653	}
7654
7655
7656	/**
7657	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7658	*/
7659	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7660	{
7661	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7662	PVM pVM = pVCpu->CTX_SUFF(pVM);
7663	Assert(!pVM->hm.s.fNestedPaging);
7664
7665	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7666	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7667	AssertRCReturn(rc, rc);
7668
7669	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7670	rc = VBOXSTRICTRC_VAL(rc2);
7671	if (RT_LIKELY(rc == VINF_SUCCESS))
7672	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7673	else
7674	{
7675	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
7676	pVmxTransient->uExitQualification, rc));
7677	}
7678	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7679	return rc;
7680	}
7681
7682
7683	/**
7684	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7685	*/
7686	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7687	{
7688	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7689	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7690	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7691	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7692	AssertRCReturn(rc, rc);
7693
7694	PVM pVM = pVCpu->CTX_SUFF(pVM);
7695	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7696	if (RT_LIKELY(rc == VINF_SUCCESS))
7697	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7698	else
7699	{
7700	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7701	rc = VERR_EM_INTERPRETER;
7702	}
7703	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7704	return rc;
7705	}
7706
7707
7708	/**
7709	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7710	*/
7711	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7712	{
7713	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7714	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7715	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7716	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7717	AssertRCReturn(rc, rc);
7718
7719	PVM pVM = pVCpu->CTX_SUFF(pVM);
7720	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7721	rc = VBOXSTRICTRC_VAL(rc2);
7722	if (RT_LIKELY( rc == VINF_SUCCESS
7723	\|\| rc == VINF_EM_HALT))
7724	{
7725	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7726	AssertRCReturn(rc3, rc3);
7727
7728	if ( rc == VINF_EM_HALT
7729	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7730	{
7731	rc = VINF_SUCCESS;
7732	}
7733	}
7734	else
7735	{
7736	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7737	rc = VERR_EM_INTERPRETER;
7738	}
7739	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7740	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7741	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7742	return rc;
7743	}
7744
7745
7746	/**
7747	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7748	*/
7749	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7750	{
7751	/*
7752	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7753	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7754	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
7755	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7756	*/
7757	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7758	pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
7759	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7760	}
7761
7762
7763	/**
7764	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7765	*/
7766	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7767	{
7768	/*
7769	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7770	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
7771	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
7772	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7773	*/
7774	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7775	pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
7776	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7777	}
7778
7779
7780	/**
7781	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7782	*/
7783	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7784	{
7785	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7786	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7787	pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
7788	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7789	}
7790
7791
7792	/**
7793	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7794	*/
7795	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7796	{
7797	/*
7798	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7799	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7800	* See Intel spec. 25.3 "Other Causes of VM-exits".
7801	*/
7802	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7803	pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
7804	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7805	}
7806
7807
7808	/**
7809	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7810	* VM-exit.
7811	*/
7812	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7813	{
7814	/*
7815	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
7816	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
7817	*
7818	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
7819	* See Intel spec. "23.8 Restrictions on VMX operation".
7820	*/
7821	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7822	return VINF_SUCCESS;
7823	}
7824
7825
7826	/**
7827	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7828	* VM-exit.
7829	*/
7830	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7831	{
7832	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7833	return VINF_EM_RESET;
7834	}
7835
7836
7837	/**
7838	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7839	*/
7840	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7841	{
7842	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7843	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7844	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7845	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7846	AssertRCReturn(rc, rc);
7847
7848	pMixedCtx->rip++;
7849	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7850	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7851	rc = VINF_SUCCESS;
7852	else
7853	rc = VINF_EM_HALT;
7854
7855	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7856	return rc;
7857	}
7858
7859
7860	/**
7861	* VM-exit handler for instructions that result in a #UD exception delivered to
7862	* the guest.
7863	*/
7864	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7865	{
7866	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7867	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7868	return VINF_SUCCESS;
7869	}
7870
7871
7872	/**
7873	* VM-exit handler for expiry of the VMX preemption timer.
7874	*/
7875	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7876	{
7877	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7878
7879	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7880	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7881
7882	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7883	PVM pVM = pVCpu->CTX_SUFF(pVM);
7884	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7885	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7886	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7887	}
7888
7889
7890	/**
7891	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7892	*/
7893	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7894	{
7895	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7896
7897	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7898	/** @todo check if XSETBV is supported by the recompiler. */
7899	return VERR_EM_INTERPRETER;
7900	}
7901
7902
7903	/**
7904	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7905	*/
7906	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7907	{
7908	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7909
7910	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7911	/** @todo implement EMInterpretInvpcid() */
7912	return VERR_EM_INTERPRETER;
7913	}
7914
7915
7916	/**
7917	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7918	* Error VM-exit.
7919	*/
7920	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7921	{
7922	uint32_t uIntrState;
7923	HMVMXHCUINTREG uHCReg;
7924	uint64_t u64Val;
7925	uint32_t u32Val;
7926
7927	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7928	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7929	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7930	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7931	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7932	AssertRCReturn(rc, rc);
7933
7934	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7935	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7936	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7937	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7938
7939	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7940	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7941	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7942	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7943	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7944	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7945	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7946	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7947	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7948	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7949	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7950	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7951
7952	PVM pVM = pVCpu->CTX_SUFF(pVM);
7953	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7954
7955	return VERR_VMX_INVALID_GUEST_STATE;
7956	}
7957
7958
7959	/**
7960	* VM-exit handler for VM-entry failure due to an MSR-load
7961	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7962	*/
7963	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7964	{
7965	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7966	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7967	}
7968
7969
7970	/**
7971	* VM-exit handler for VM-entry failure due to a machine-check event
7972	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7973	*/
7974	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7975	{
7976	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7977	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7978	}
7979
7980
7981	/**
7982	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7983	* theory.
7984	*/
7985	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7986	{
7987	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7988	return VERR_VMX_UNDEFINED_EXIT_CODE;
7989	}
7990
7991
7992	/**
7993	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7994	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7995	* Conditional VM-exit.
7996	*/
7997	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7998	{
7999	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8000
8001	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
8002	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
8003	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
8004	return VERR_EM_INTERPRETER;
8005	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8006	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8007	}
8008
8009
8010	/**
8011	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
8012	*/
8013	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8014	{
8015	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8016
8017	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
8018	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
8019	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
8020	return VERR_EM_INTERPRETER;
8021	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8022	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8023	}
8024
8025
8026	/**
8027	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
8028	*/
8029	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8030	{
8031	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8032
8033	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
8034	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8035	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8036	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8037	AssertRCReturn(rc, rc);
8038
8039	PVM pVM = pVCpu->CTX_SUFF(pVM);
8040	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8041	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
8042	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
8043	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
8044
8045	if (RT_LIKELY(rc == VINF_SUCCESS))
8046	{
8047	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8048	Assert(pVmxTransient->cbInstr == 2);
8049	}
8050	return rc;
8051	}
8052
8053
8054	/**
8055	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
8056	*/
8057	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8058	{
8059	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8060	PVM pVM = pVCpu->CTX_SUFF(pVM);
8061	int rc = VINF_SUCCESS;
8062
8063	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
8064	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8065	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8066	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8067	AssertRCReturn(rc, rc);
8068	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
8069
8070	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8071	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
8072	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
8073
8074	if (RT_LIKELY(rc == VINF_SUCCESS))
8075	{
8076	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8077
8078	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
8079	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
8080	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
8081	{
8082	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
8083	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
8084	EMInterpretWrmsr() changes it. */
8085	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8086	}
8087	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
8088	{
8089	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8090	AssertRCReturn(rc, rc);
8091	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
8092	}
8093	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
8094	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8095
8096	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
8097	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8098	{
8099	switch (pMixedCtx->ecx)
8100	{
8101	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
8102	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
8103	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
8104	case MSR_K8_FS_BASE: /* no break */
8105	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
8106	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
8107	}
8108	}
8109	#ifdef VBOX_STRICT
8110	else
8111	{
8112	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
8113	switch (pMixedCtx->ecx)
8114	{
8115	case MSR_IA32_SYSENTER_CS:
8116	case MSR_IA32_SYSENTER_EIP:
8117	case MSR_IA32_SYSENTER_ESP:
8118	case MSR_K8_FS_BASE:
8119	case MSR_K8_GS_BASE:
8120	{
8121	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
8122	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8123	}
8124
8125	case MSR_K8_LSTAR:
8126	case MSR_K6_STAR:
8127	case MSR_K8_SF_MASK:
8128	case MSR_K8_TSC_AUX:
8129	case MSR_K8_KERNEL_GS_BASE:
8130	{
8131	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
8132	pMixedCtx->ecx));
8133	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8134	}
8135	}
8136	}
8137	#endif /* VBOX_STRICT */
8138	}
8139	return rc;
8140	}
8141
8142
8143	/**
8144	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
8145	*/
8146	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8147	{
8148	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8149
8150	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
8151	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
8152	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
8153	return VERR_EM_INTERPRETER;
8154	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8155	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8156	}
8157
8158
8159	/**
8160	* VM-exit handler for when the TPR value is lowered below the specified
8161	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
8162	*/
8163	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8164	{
8165	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8166	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
8167
8168	/*
8169	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
8170	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
8171	* resume guest execution.
8172	*/
8173	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8174	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
8175	return VINF_SUCCESS;
8176	}
8177
8178
8179	/**
8180	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
8181	* VM-exit.
8182	*
8183	* @retval VINF_SUCCESS when guest execution can continue.
8184	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
8185	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
8186	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
8187	* recompiler.
8188	*/
8189	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8190	{
8191	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8192	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
8193	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8194	AssertRCReturn(rc, rc);
8195
8196	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
8197	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
8198	PVM pVM = pVCpu->CTX_SUFF(pVM);
8199	switch (uAccessType)
8200	{
8201	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
8202	{
8203	#if 0
8204	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
8205	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8206	#else
8207	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8208	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8209	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8210	#endif
8211	AssertRCReturn(rc, rc);
8212
8213	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8214	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
8215	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
8216	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
8217
8218	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
8219	{
8220	case 0: /* CR0 */
8221	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
8222	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8223	break;
8224	case 2: /* C2 **/
8225	/* Nothing to do here, CR2 it's not part of the VMCS. */
8226	break;
8227	case 3: /* CR3 */
8228	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
8229	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
8230	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
8231	break;
8232	case 4: /* CR4 */
8233	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
8234	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
8235	break;
8236	case 8: /* CR8 */
8237	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8238	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
8239	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8240	break;
8241	default:
8242	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
8243	break;
8244	}
8245
8246	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8247	break;
8248	}
8249
8250	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
8251	{
8252	/* EMInterpretCRxRead() requires EFER MSR, CS. */
8253	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8254	AssertRCReturn(rc, rc);
8255	Assert( !pVM->hm.s.fNestedPaging
8256	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
8257	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
8258
8259	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8260	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
8261	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8262
8263	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8264	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
8265	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
8266	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8267	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8268	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
8269	break;
8270	}
8271
8272	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
8273	{
8274	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8275	AssertRCReturn(rc, rc);
8276	rc = EMInterpretCLTS(pVM, pVCpu);
8277	AssertRCReturn(rc, rc);
8278	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8279	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
8280	Log4(("CRX CLTS write rc=%d\n", rc));
8281	break;
8282	}
8283
8284	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
8285	{
8286	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8287	AssertRCReturn(rc, rc);
8288	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
8289	if (RT_LIKELY(rc == VINF_SUCCESS))
8290	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8291	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
8292	Log4(("CRX LMSW write rc=%d\n", rc));
8293	break;
8294	}
8295
8296	default:
8297	{
8298	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
8299	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8300	}
8301	}
8302
8303	/* Validate possible error codes. */
8304	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
8305	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8306	if (RT_SUCCESS(rc))
8307	{
8308	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8309	AssertRCReturn(rc2, rc2);
8310	}
8311
8312	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8313	return rc;
8314	}
8315
8316
8317	/**
8318	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8319	* VM-exit.
8320	*/
8321	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8322	{
8323	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8324	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8325
8326	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8327	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8328	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8329	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8330	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8331	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8332	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8333	AssertRCReturn(rc, rc);
8334
8335	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8336
8337	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8338	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8339	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8340	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8341	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8342	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
8343	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
8344
8345	/* I/O operation lookup arrays. */
8346	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
8347	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8348
8349	const uint32_t cbSize = s_aIOSize[uIOWidth];
8350	const uint32_t cbInstr = pVmxTransient->cbInstr;
8351	PVM pVM = pVCpu->CTX_SUFF(pVM);
8352	if (fIOString)
8353	{
8354	/* INS/OUTS - I/O String instruction. */
8355	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8356	/** @todo for now manually disassemble later optimize by getting the fields from
8357	* the VMCS. VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8358	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8359	* segment prefix info. */
8360	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8361	if (RT_SUCCESS(rc))
8362	{
8363	if (fIOWrite)
8364	{
8365	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8366	(DISCPUMODE)pDis->uAddrMode, cbSize);
8367	rc = VBOXSTRICTRC_VAL(rc2);
8368	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8369	}
8370	else
8371	{
8372	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8373	(DISCPUMODE)pDis->uAddrMode, cbSize);
8374	rc = VBOXSTRICTRC_VAL(rc2);
8375	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8376	}
8377	}
8378	else
8379	{
8380	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8381	rc = VINF_EM_RAW_EMULATE_INSTR;
8382	}
8383	}
8384	else
8385	{
8386	/* IN/OUT - I/O instruction. */
8387	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8388	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8389	if (fIOWrite)
8390	{
8391	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8392	rc = VBOXSTRICTRC_VAL(rc2);
8393	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8394	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8395	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8396	}
8397	else
8398	{
8399	uint32_t u32Result = 0;
8400	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8401	rc = VBOXSTRICTRC_VAL(rc2);
8402	if (IOM_SUCCESS(rc))
8403	{
8404	/* Save result of I/O IN instr. in AL/AX/EAX. */
8405	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8406	}
8407	else if (rc == VINF_IOM_R3_IOPORT_READ)
8408	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8409	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8410	}
8411	}
8412
8413	if (IOM_SUCCESS(rc))
8414	{
8415	pMixedCtx->rip += cbInstr;
8416	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8417	if (RT_LIKELY(rc == VINF_SUCCESS))
8418	{
8419	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8420	AssertRCReturn(rc, rc);
8421
8422	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8423	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8424	{
8425	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8426	for (unsigned i = 0; i < 4; i++)
8427	{
8428	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8429	if ( ( uIOPort >= pMixedCtx->dr[i]
8430	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8431	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8432	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8433	{
8434	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8435	uint64_t uDR6 = ASMGetDR6();
8436
8437	/* Clear all breakpoint status flags and set the one we just hit. */
8438	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8439	uDR6 \|= (uint64_t)RT_BIT(i);
8440
8441	/*
8442	* Note: AMD64 Architecture Programmer's Manual 13.1:
8443	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8444	* be cleared by software after the contents have been read.
8445	*/
8446	ASMSetDR6(uDR6);
8447
8448	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8449	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8450
8451	/* Paranoia. */
8452	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits MBZ. */
8453	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8454	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8455
8456	/* Resync DR7 */
8457	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8458	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8459
8460	/* Set #DB to be injected into the VM and continue guest execution. */
8461	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8462	break;
8463	}
8464	}
8465	}
8466	}
8467	}
8468
8469	#ifdef DEBUG
8470	if (rc == VINF_IOM_R3_IOPORT_READ)
8471	Assert(!fIOWrite);
8472	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8473	Assert(fIOWrite);
8474	else
8475	{
8476	AssertMsg( RT_FAILURE(rc)
8477	\|\| rc == VINF_SUCCESS
8478	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8479	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8480	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8481	}
8482	#endif
8483
8484	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8485	return rc;
8486	}
8487
8488
8489	/**
8490	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8491	* VM-exit.
8492	*/
8493	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8494	{
8495	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8496
8497	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8498	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8499	AssertRCReturn(rc, rc);
8500	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8501	{
8502	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8503	AssertRCReturn(rc, rc);
8504	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8505	{
8506	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8507
8508	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8509	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8510	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8511	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8512	{
8513	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8514	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8515
8516	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8517	Assert(!pVCpu->hm.s.Event.fPending);
8518	pVCpu->hm.s.Event.fPending = true;
8519	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8520	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8521	AssertRCReturn(rc, rc);
8522	if (fErrorCodeValid)
8523	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8524	else
8525	pVCpu->hm.s.Event.u32ErrCode = 0;
8526	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8527	&& uVector == X86_XCPT_PF)
8528	{
8529	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8530	}
8531
8532	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8533	}
8534	}
8535	}
8536
8537	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8538	* emulation. */
8539	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8540	return VERR_EM_INTERPRETER;
8541	}
8542
8543
8544	/**
8545	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8546	*/
8547	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8548	{
8549	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8550	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8551	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8552	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8553	AssertRCReturn(rc, rc);
8554	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8555	return VINF_EM_DBG_STOP;
8556	}
8557
8558
8559	/**
8560	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8561	*/
8562	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8563	{
8564	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8565
8566	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8567	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8568	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8569	return VINF_SUCCESS;
8570	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8571	return rc;
8572
8573	#if 0
8574	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8575	* just sync the whole thing. */
8576	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8577	#else
8578	/* Aggressive state sync. for now. */
8579	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8580	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8581	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8582	#endif
8583	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8584	AssertRCReturn(rc, rc);
8585
8586	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8587	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8588	switch (uAccessType)
8589	{
8590	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8591	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8592	{
8593	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8594	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8595	{
8596	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8597	}
8598
8599	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8600	GCPhys &= PAGE_BASE_GC_MASK;
8601	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8602	PVM pVM = pVCpu->CTX_SUFF(pVM);
8603	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
8604	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8605
8606	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8607	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8608	CPUMCTX2CORE(pMixedCtx), GCPhys);
8609	rc = VBOXSTRICTRC_VAL(rc2);
8610	Log4(("ApicAccess rc=%d\n", rc));
8611	if ( rc == VINF_SUCCESS
8612	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8613	\|\| rc == VERR_PAGE_NOT_PRESENT)
8614	{
8615	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8616	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8617	rc = VINF_SUCCESS;
8618	}
8619	break;
8620	}
8621
8622	default:
8623	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
8624	rc = VINF_EM_RAW_EMULATE_INSTR;
8625	break;
8626	}
8627
8628	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8629	return rc;
8630	}
8631
8632
8633	/**
8634	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8635	* VM-exit.
8636	*/
8637	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8638	{
8639	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8640
8641	/* We should -not- get this VM-exit if the guest is debugging. */
8642	if (CPUMIsGuestDebugStateActive(pVCpu))
8643	{
8644	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8645	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8646	}
8647
8648	int rc = VERR_INTERNAL_ERROR_5;
8649	if ( !DBGFIsStepping(pVCpu)
8650	&& !CPUMIsHyperDebugStateActive(pVCpu))
8651	{
8652	/* Don't intercept MOV DRx. */
8653	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8654	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8655	AssertRCReturn(rc, rc);
8656
8657	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8658	PVM pVM = pVCpu->CTX_SUFF(pVM);
8659	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8660	AssertRC(rc);
8661	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8662
8663	#ifdef VBOX_WITH_STATISTICS
8664	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8665	AssertRCReturn(rc, rc);
8666	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8667	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8668	else
8669	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8670	#endif
8671	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8672	return VINF_SUCCESS;
8673	}
8674
8675	/*
8676	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8677	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8678	*/
8679	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8680	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8681	AssertRCReturn(rc, rc);
8682
8683	PVM pVM = pVCpu->CTX_SUFF(pVM);
8684	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8685	{
8686	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8687	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8688	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8689	if (RT_SUCCESS(rc))
8690	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8691	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8692	}
8693	else
8694	{
8695	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8696	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8697	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8698	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8699	}
8700
8701	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8702	if (RT_SUCCESS(rc))
8703	{
8704	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8705	AssertRCReturn(rc2, rc2);
8706	}
8707	return rc;
8708	}
8709
8710
8711	/**
8712	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8713	* Conditional VM-exit.
8714	*/
8715	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8716	{
8717	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8718	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8719
8720	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8721	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8722	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8723	return VINF_SUCCESS;
8724	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8725	return rc;
8726
8727	RTGCPHYS GCPhys = 0;
8728	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8729
8730	#if 0
8731	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8732	#else
8733	/* Aggressive state sync. for now. */
8734	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8735	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8736	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8737	#endif
8738	AssertRCReturn(rc, rc);
8739
8740	/*
8741	* If we succeed, resume guest execution.
8742	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8743	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8744	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8745	* weird case. See @bugref{6043}.
8746	*/
8747	PVM pVM = pVCpu->CTX_SUFF(pVM);
8748	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8749	rc = VBOXSTRICTRC_VAL(rc2);
8750	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8751	if ( rc == VINF_SUCCESS
8752	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8753	\|\| rc == VERR_PAGE_NOT_PRESENT)
8754	{
8755	/* Successfully handled MMIO operation. */
8756	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8757	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8758	rc = VINF_SUCCESS;
8759	}
8760	return rc;
8761	}
8762
8763
8764	/**
8765	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8766	* VM-exit.
8767	*/
8768	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8769	{
8770	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8771	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8772
8773	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8774	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8775	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8776	return VINF_SUCCESS;
8777	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8778	return rc;
8779
8780	RTGCPHYS GCPhys = 0;
8781	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8782	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8783	#if 0
8784	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8785	#else
8786	/* Aggressive state sync. for now. */
8787	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8788	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8789	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8790	#endif
8791	AssertRCReturn(rc, rc);
8792
8793	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8794	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
8795
8796	RTGCUINT uErrorCode = 0;
8797	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8798	uErrorCode \|= X86_TRAP_PF_ID;
8799	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8800	uErrorCode \|= X86_TRAP_PF_RW;
8801	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8802	uErrorCode \|= X86_TRAP_PF_P;
8803
8804	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8805
8806	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
8807	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8808
8809	/* Handle the pagefault trap for the nested shadow table. */
8810	PVM pVM = pVCpu->CTX_SUFF(pVM);
8811	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8812	TRPMResetTrap(pVCpu);
8813
8814	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8815	if ( rc == VINF_SUCCESS
8816	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8817	\|\| rc == VERR_PAGE_NOT_PRESENT)
8818	{
8819	/* Successfully synced our nested page tables. */
8820	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8821	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8822	return VINF_SUCCESS;
8823	}
8824
8825	Log4(("EPT return to ring-3 rc=%d\n"));
8826	return rc;
8827	}
8828
8829	/** @} */
8830
8831	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8832	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8833	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8834
8835	/** @name VM-exit exception handlers.
8836	* @{
8837	*/
8838
8839	/**
8840	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8841	*/
8842	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8843	{
8844	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8845	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8846
8847	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8848	AssertRCReturn(rc, rc);
8849
8850	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8851	{
8852	/* Old-style FPU error reporting needs some extra work. */
8853	/** @todo don't fall back to the recompiler, but do it manually. */
8854	return VERR_EM_INTERPRETER;
8855	}
8856
8857	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8858	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8859	return rc;
8860	}
8861
8862
8863	/**
8864	* VM-exit exception handler for #BP (Breakpoint exception).
8865	*/
8866	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8867	{
8868	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8869	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8870
8871	/** @todo Try optimize this by not saving the entire guest state unless
8872	* really needed. */
8873	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8874	AssertRCReturn(rc, rc);
8875
8876	PVM pVM = pVCpu->CTX_SUFF(pVM);
8877	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8878	if (rc == VINF_EM_RAW_GUEST_TRAP)
8879	{
8880	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8881	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8882	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8883	AssertRCReturn(rc, rc);
8884
8885	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8886	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8887	}
8888
8889	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8890	return rc;
8891	}
8892
8893
8894	/**
8895	* VM-exit exception handler for #DB (Debug exception).
8896	*/
8897	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8898	{
8899	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8900	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8901
8902	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8903	AssertRCReturn(rc, rc);
8904
8905	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8906	uint64_t uDR6 = X86_DR6_INIT_VAL;
8907	uDR6 \|= (pVmxTransient->uExitQualification
8908	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8909	PVM pVM = pVCpu->CTX_SUFF(pVM);
8910	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8911	if (rc == VINF_EM_RAW_GUEST_TRAP)
8912	{
8913	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8914	pMixedCtx->dr[6] = uDR6;
8915
8916	if (CPUMIsGuestDebugStateActive(pVCpu))
8917	ASMSetDR6(pMixedCtx->dr[6]);
8918
8919	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8920
8921	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8922	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8923
8924	/* Paranoia. */
8925	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits MBZ. */
8926	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8927	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8928
8929	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
8930	AssertRCReturn(rc,rc);
8931
8932	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8933	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8934	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8935	AssertRCReturn(rc2, rc2);
8936	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8937	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8938	rc = VINF_SUCCESS;
8939	}
8940
8941	return rc;
8942	}
8943
8944
8945	/**
8946	* VM-exit exception handler for #NM (Device-not-available exception: floating
8947	* point exception).
8948	*/
8949	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8950	{
8951	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8952
8953	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8954	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8955	#endif
8956
8957	/* We require CR0 and EFER. EFER is always up-to-date. */
8958	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8959	AssertRCReturn(rc, rc);
8960
8961	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8962	PVM pVM = pVCpu->CTX_SUFF(pVM);
8963	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8964	if (rc == VINF_SUCCESS)
8965	{
8966	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8967	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8968	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8969	return VINF_SUCCESS;
8970	}
8971
8972	/* Forward #NM to the guest. */
8973	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8974	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8975	AssertRCReturn(rc, rc);
8976	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8977	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8978	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8979	return rc;
8980	}
8981
8982
8983	/**
8984	* VM-exit exception handler for #GP (General-protection exception).
8985	*
8986	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8987	*/
8988	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8989	{
8990	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8991	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8992
8993	int rc = VERR_INTERNAL_ERROR_5;
8994	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8995	{
8996	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8997	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8998	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8999	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9000	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9001	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9002	AssertRCReturn(rc, rc);
9003	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
9004	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
9005	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9006	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9007	return rc;
9008	#else
9009	/* We don't intercept #GP. */
9010	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
9011	return VERR_VMX_UNEXPECTED_EXCEPTION;
9012	#endif
9013	}
9014
9015	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9016	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
9017
9018	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
9019	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9020	AssertRCReturn(rc, rc);
9021
9022	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9023	uint32_t cbOp = 0;
9024	PVM pVM = pVCpu->CTX_SUFF(pVM);
9025	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
9026	if (RT_SUCCESS(rc))
9027	{
9028	rc = VINF_SUCCESS;
9029	Assert(cbOp == pDis->cbInstr);
9030	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
9031	switch (pDis->pCurInstr->uOpcode)
9032	{
9033	case OP_CLI:
9034	{
9035	pMixedCtx->eflags.Bits.u1IF = 0;
9036	pMixedCtx->rip += pDis->cbInstr;
9037	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
9038	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
9039	break;
9040	}
9041
9042	case OP_STI:
9043	{
9044	pMixedCtx->eflags.Bits.u1IF = 1;
9045	pMixedCtx->rip += pDis->cbInstr;
9046	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
9047	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9048	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
9049	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
9050	break;
9051	}
9052
9053	case OP_HLT:
9054	{
9055	rc = VINF_EM_HALT;
9056	pMixedCtx->rip += pDis->cbInstr;
9057	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9058	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9059	break;
9060	}
9061
9062	case OP_POPF:
9063	{
9064	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
9065	uint32_t cbParm = 0;
9066	uint32_t uMask = 0;
9067	if (pDis->fPrefix & DISPREFIX_OPSIZE)
9068	{
9069	cbParm = 4;
9070	uMask = 0xffffffff;
9071	}
9072	else
9073	{
9074	cbParm = 2;
9075	uMask = 0xffff;
9076	}
9077
9078	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
9079	RTGCPTR GCPtrStack = 0;
9080	X86EFLAGS uEflags;
9081	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9082	&GCPtrStack);
9083	if (RT_SUCCESS(rc))
9084	{
9085	Assert(sizeof(uEflags.u32) >= cbParm);
9086	uEflags.u32 = 0;
9087	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
9088	}
9089	if (RT_FAILURE(rc))
9090	{
9091	rc = VERR_EM_INTERPRETER;
9092	break;
9093	}
9094	Log4(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
9095	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9096	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
9097	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
9098	pMixedCtx->eflags.Bits.u1RF = 0;
9099	pMixedCtx->esp += cbParm;
9100	pMixedCtx->esp &= uMask;
9101	pMixedCtx->rip += pDis->cbInstr;
9102	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
9103	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
9104	break;
9105	}
9106
9107	case OP_PUSHF:
9108	{
9109	uint32_t cbParm = 0;
9110	uint32_t uMask = 0;
9111	if (pDis->fPrefix & DISPREFIX_OPSIZE)
9112	{
9113	cbParm = 4;
9114	uMask = 0xffffffff;
9115	}
9116	else
9117	{
9118	cbParm = 2;
9119	uMask = 0xffff;
9120	}
9121
9122	/* Get the stack pointer & push the contents of eflags onto the stack. */
9123	RTGCPTR GCPtrStack = 0;
9124	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
9125	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
9126	if (RT_FAILURE(rc))
9127	{
9128	rc = VERR_EM_INTERPRETER;
9129	break;
9130	}
9131	X86EFLAGS uEflags;
9132	uEflags = pMixedCtx->eflags;
9133	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
9134	uEflags.Bits.u1RF = 0;
9135	uEflags.Bits.u1VM = 0;
9136
9137	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
9138	if (RT_FAILURE(rc))
9139	{
9140	rc = VERR_EM_INTERPRETER;
9141	break;
9142	}
9143	Log4(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
9144	pMixedCtx->esp -= cbParm;
9145	pMixedCtx->esp &= uMask;
9146	pMixedCtx->rip += pDis->cbInstr;
9147	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
9148	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
9149	break;
9150	}
9151
9152	case OP_IRET:
9153	{
9154	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
9155	* instruction reference. */
9156	RTGCPTR GCPtrStack = 0;
9157	uint32_t uMask = 0xffff;
9158	uint16_t aIretFrame[3];
9159	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
9160	{
9161	rc = VERR_EM_INTERPRETER;
9162	break;
9163	}
9164	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9165	&GCPtrStack);
9166	if (RT_SUCCESS(rc))
9167	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
9168	if (RT_FAILURE(rc))
9169	{
9170	rc = VERR_EM_INTERPRETER;
9171	break;
9172	}
9173	pMixedCtx->eip = 0;
9174	pMixedCtx->ip = aIretFrame[0];
9175	pMixedCtx->cs.Sel = aIretFrame[1];
9176	pMixedCtx->cs.ValidSel = aIretFrame[1];
9177	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
9178	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9179	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
9180	pMixedCtx->sp += sizeof(aIretFrame);
9181	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
9182	\| HM_CHANGED_GUEST_RFLAGS;
9183	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
9184	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
9185	break;
9186	}
9187
9188	case OP_INT:
9189	{
9190	uint16_t uVector = pDis->Param1.uValue & 0xff;
9191	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
9192	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9193	break;
9194	}
9195
9196	case OP_INTO:
9197	{
9198	if (pMixedCtx->eflags.Bits.u1OF)
9199	{
9200	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
9201	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9202	}
9203	break;
9204	}
9205
9206	default:
9207	{
9208	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
9209	EMCODETYPE_SUPERVISOR);
9210	rc = VBOXSTRICTRC_VAL(rc2);
9211	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
9212	Log4(("#GP rc=%Rrc\n", rc));
9213	break;
9214	}
9215	}
9216	}
9217	else
9218	rc = VERR_EM_INTERPRETER;
9219
9220	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
9221	("#GP Unexpected rc=%Rrc\n", rc));
9222	return rc;
9223	}
9224
9225
9226	/**
9227	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
9228	* the exception reported in the VMX transient structure back into the VM.
9229	*
9230	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
9231	* up-to-date.
9232	*/
9233	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9234	{
9235	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9236
9237	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
9238	hmR0VmxCheckExitDueToEventDelivery(). */
9239	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9240	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9241	AssertRCReturn(rc, rc);
9242	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
9243
9244	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9245	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9246	return VINF_SUCCESS;
9247	}
9248
9249
9250	/**
9251	* VM-exit exception handler for #PF (Page-fault exception).
9252	*/
9253	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9254	{
9255	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9256	PVM pVM = pVCpu->CTX_SUFF(pVM);
9257	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9258	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9259	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9260	AssertRCReturn(rc, rc);
9261
9262	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
9263	if (pVM->hm.s.fNestedPaging)
9264	{
9265	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
9266	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
9267	{
9268	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9269	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9270	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
9271	}
9272	else
9273	{
9274	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9275	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9276	Log4(("Pending #DF due to vectoring #PF. NP\n"));
9277	}
9278	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9279	return rc;
9280	}
9281	#else
9282	Assert(!pVM->hm.s.fNestedPaging);
9283	#endif
9284
9285	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9286	AssertRCReturn(rc, rc);
9287
9288	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
9289	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9290
9291	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9292	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9293	(RTGCPTR)pVmxTransient->uExitQualification);
9294
9295	Log4(("#PF: rc=%Rrc\n", rc));
9296	if (rc == VINF_SUCCESS)
9297	{
9298	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9299	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9300	* memory? We don't update the whole state here... */
9301	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9302	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9303	TRPMResetTrap(pVCpu);
9304	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9305	return rc;
9306	}
9307	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9308	{
9309	if (!pVmxTransient->fVectoringPF)
9310	{
9311	/* It's a guest page fault and needs to be reflected to the guest. */
9312	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9313	TRPMResetTrap(pVCpu);
9314	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9315	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9316	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9317	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9318	}
9319	else
9320	{
9321	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9322	TRPMResetTrap(pVCpu);
9323	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
9324	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9325	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
9326	}
9327
9328	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9329	return VINF_SUCCESS;
9330	}
9331
9332	TRPMResetTrap(pVCpu);
9333	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9334	return rc;
9335	}
9336
9337	/** @} */
9338

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 47389

Download in other formats: