HMVMXR0.cpp@ 47247

Last change on this file since 47247 was 47247, checked in by vboxsync, 12 years ago
HMVMX: preserve SS.DPL and CS.L/D/G when the 'unusable' bit is set. SS.DPL = CPL, so we absolutely must preserve that.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 390.6 KB

Line
1	/* $Id: HMVMXR0.cpp 47247 2013-07-19 10:01:36Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#if defined(RT_ARCH_AMD64)
49	# define HMVMX_IS_64BIT_HOST_MODE() (true)
50	typedef RTHCUINTREG HMVMXHCUINTREG;
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	typedef uint64_t HMVMXHCUINTREG;
55	#else
56	# define HMVMX_IS_64BIT_HOST_MODE() (false)
57	typedef RTHCUINTREG HMVMXHCUINTREG;
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** Determine which tagged-TLB flush handler to use. */
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69	/** @name Updated-guest-state flags.
70	* @{ */
71	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
83	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
84	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
86	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
88	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
89	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
90	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92	\| HMVMX_UPDATED_GUEST_RSP \
93	\| HMVMX_UPDATED_GUEST_RFLAGS \
94	\| HMVMX_UPDATED_GUEST_CR0 \
95	\| HMVMX_UPDATED_GUEST_CR3 \
96	\| HMVMX_UPDATED_GUEST_CR4 \
97	\| HMVMX_UPDATED_GUEST_GDTR \
98	\| HMVMX_UPDATED_GUEST_IDTR \
99	\| HMVMX_UPDATED_GUEST_LDTR \
100	\| HMVMX_UPDATED_GUEST_TR \
101	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102	\| HMVMX_UPDATED_GUEST_DEBUG \
103	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
104	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
106	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
108	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
109	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
110	\| HMVMX_UPDATED_GUEST_APIC_STATE)
111	/** @} */
112
113	/**
114	* Flags to skip redundant reads of some common VMCS fields that are not part of
115	* the guest-CPU state but are in the transient structure.
116	*/
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123
124	/**
125	* @name Exception bitmap mask for real-mode guests (real-on-v86).
126	*
127	* We need to intercept all exceptions manually (except #PF). #NM is also
128	* handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be
129	* intercepted even in real-mode if we have Nested Paging support.
130	*/
131	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
132	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
133	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
134	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
135	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
136	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
137	\| RT_BIT(X86_XCPT_XF))
138	/** @} */
139
140	/**
141	* @name Exception bitmap mask for all contributory exceptions.
142	*
143	* Page fault is deliberately excluded here as it's conditional as to whether
144	* it's contributory or benign. Page faults are handled separately.
145	*/
146	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
147	\| RT_BIT(X86_XCPT_DE))
148	/** @} */
149
150	/** Maximum VM-instruction error number. */
151	#define HMVMX_INSTR_ERROR_MAX 28
152
153	/** Profiling macro. */
154	#ifdef HM_PROFILE_EXIT_DISPATCH
155	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
156	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
157	#else
158	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
159	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
160	#endif
161
162
163	/*******************************************************************************
164	* Structures and Typedefs *
165	*******************************************************************************/
166	/** @name VMX transient.
167	*
168	* A state structure for holding miscellaneous information across
169	* VMX non-root operation and restored after the transition.
170	*
171	* @{ */
172	typedef struct VMXTRANSIENT
173	{
174	/** The host's rflags/eflags. */
175	RTCCUINTREG uEFlags;
176	#if HC_ARCH_BITS == 32
177	uint32_t u32Alignment0;
178	#endif
179	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
180	uint64_t u64LStarMsr;
181	/** The guest's TPR value used for TPR shadowing. */
182	uint8_t u8GuestTpr;
183	/** Alignment. */
184	uint8_t abAlignment0[7];
185
186	/** The basic VM-exit reason. */
187	uint16_t uExitReason;
188	/** Alignment. */
189	uint16_t u16Alignment0;
190	/** The VM-exit interruption error code. */
191	uint32_t uExitIntrErrorCode;
192	/** The VM-exit exit qualification. */
193	uint64_t uExitQualification;
194
195	/** The VM-exit interruption-information field. */
196	uint32_t uExitIntrInfo;
197	/** The VM-exit instruction-length field. */
198	uint32_t cbInstr;
199	/** Whether the VM-entry failed or not. */
200	bool fVMEntryFailed;
201	/** Alignment. */
202	uint8_t abAlignment1[7];
203
204	/** The VM-entry interruption-information field. */
205	uint32_t uEntryIntrInfo;
206	/** The VM-entry exception error code field. */
207	uint32_t uEntryXcptErrorCode;
208	/** The VM-entry instruction length field. */
209	uint32_t cbEntryInstr;
210
211	/** IDT-vectoring information field. */
212	uint32_t uIdtVectoringInfo;
213	/** IDT-vectoring error code. */
214	uint32_t uIdtVectoringErrorCode;
215
216	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
217	uint32_t fVmcsFieldsRead;
218	/** Whether TSC-offsetting should be setup before VM-entry. */
219	bool fUpdateTscOffsettingAndPreemptTimer;
220	/** Whether the VM-exit was caused by a page-fault during delivery of a
221	* contributary exception or a page-fault. */
222	bool fVectoringPF;
223	} VMXTRANSIENT, *PVMXTRANSIENT;
224	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
225	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
226	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
227	/** @} */
228
229
230	/**
231	* MSR-bitmap read permissions.
232	*/
233	typedef enum VMXMSREXITREAD
234	{
235	/** Reading this MSR causes a VM-exit. */
236	VMXMSREXIT_INTERCEPT_READ = 0xb,
237	/** Reading this MSR does not cause a VM-exit. */
238	VMXMSREXIT_PASSTHRU_READ
239	} VMXMSREXITREAD;
240
241	/**
242	* MSR-bitmap write permissions.
243	*/
244	typedef enum VMXMSREXITWRITE
245	{
246	/** Writing to this MSR causes a VM-exit. */
247	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
248	/** Writing to this MSR does not cause a VM-exit. */
249	VMXMSREXIT_PASSTHRU_WRITE
250	} VMXMSREXITWRITE;
251
252
253	/*******************************************************************************
254	* Internal Functions *
255	*******************************************************************************/
256	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
257	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
258	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
259	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
260	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
261	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
262	#endif
263	#ifndef HMVMX_USE_FUNCTION_TABLE
264	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
265	#define HMVMX_EXIT_DECL static int
266	#else
267	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
268	#endif
269
270	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
307	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
309	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314
315	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
316	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
317	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
318	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
319	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
320	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
321	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
322
323
324	/*******************************************************************************
325	* Global Variables *
326	*******************************************************************************/
327	#ifdef HMVMX_USE_FUNCTION_TABLE
328	/**
329	* VM-exit handler.
330	*
331	* @returns VBox status code.
332	* @param pVCpu Pointer to the VMCPU.
333	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
334	* out-of-sync. Make sure to update the required
335	* fields before using them.
336	* @param pVmxTransient Pointer to the VMX-transient structure.
337	*/
338	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
339	/** Pointer to VM-exit handler. */
340	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
341
342	/**
343	* VMX_EXIT dispatch table.
344	*/
345	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
346	{
347	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
348	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
349	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
350	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
351	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
352	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
353	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
354	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
355	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
356	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
357	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
358	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
359	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
360	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
361	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
362	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
363	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
364	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
365	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
366	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
367	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
368	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
369	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
370	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
371	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
372	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
373	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
374	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
375	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
376	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
377	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
378	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
379	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
380	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
381	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
382	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
383	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
384	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
385	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
386	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
387	/* 40 UNDEFINED */ hmR0VmxExitPause,
388	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
389	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
390	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
391	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
392	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
393	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
394	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
395	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
396	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
397	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
398	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
399	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
400	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
401	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
402	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
403	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
404	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
405	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
406	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
407	};
408	#endif /* HMVMX_USE_FUNCTION_TABLE */
409
410	#ifdef VBOX_STRICT
411	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
412	{
413	/* 0 */ "(Not Used)",
414	/* 1 */ "VMCALL executed in VMX root operation.",
415	/* 2 */ "VMCLEAR with invalid physical address.",
416	/* 3 */ "VMCLEAR with VMXON pointer.",
417	/* 4 */ "VMLAUNCH with non-clear VMCS.",
418	/* 5 */ "VMRESUME with non-launched VMCS.",
419	/* 6 */ "VMRESUME after VMXOFF",
420	/* 7 */ "VM entry with invalid control fields.",
421	/* 8 */ "VM entry with invalid host state fields.",
422	/* 9 */ "VMPTRLD with invalid physical address.",
423	/* 10 */ "VMPTRLD with VMXON pointer.",
424	/* 11 */ "VMPTRLD with incorrect revision identifier.",
425	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
426	/* 13 */ "VMWRITE to read-only VMCS component.",
427	/* 14 */ "(Not Used)",
428	/* 15 */ "VMXON executed in VMX root operation.",
429	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
430	/* 17 */ "VM entry with non-launched executing VMCS.",
431	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
432	/* 19 */ "VMCALL with non-clear VMCS.",
433	/* 20 */ "VMCALL with invalid VM-exit control fields.",
434	/* 21 */ "(Not Used)",
435	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
436	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
437	/* 24 */ "VMCALL with invalid SMM-monitor features.",
438	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
439	/* 26 */ "VM entry with events blocked by MOV SS.",
440	/* 27 */ "(Not Used)",
441	/* 28 */ "Invalid operand to INVEPT/INVVPID."
442	};
443	#endif /* VBOX_STRICT */
444
445
446
447	/**
448	* Updates the VM's last error record. If there was a VMX instruction error,
449	* reads the error data from the VMCS and updates VCPU's last error record as
450	* well.
451	*
452	* @param pVM Pointer to the VM.
453	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
454	* VERR_VMX_UNABLE_TO_START_VM or
455	* VERR_VMX_INVALID_VMCS_FIELD).
456	* @param rc The error code.
457	*/
458	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
459	{
460	AssertPtr(pVM);
461	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
462	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
463	{
464	AssertPtrReturnVoid(pVCpu);
465	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
466	}
467	pVM->hm.s.lLastError = rc;
468	}
469
470
471	/**
472	* Reads the VM-entry interruption-information field from the VMCS into the VMX
473	* transient structure.
474	*
475	* @returns VBox status code.
476	* @param pVmxTransient Pointer to the VMX transient structure.
477	*
478	* @remarks No-long-jump zone!!!
479	*/
480	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
481	{
482	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
483	AssertRCReturn(rc, rc);
484	return VINF_SUCCESS;
485	}
486
487
488	/**
489	* Reads the VM-entry exception error code field from the VMCS into
490	* the VMX transient structure.
491	*
492	* @returns VBox status code.
493	* @param pVmxTransient Pointer to the VMX transient structure.
494	*
495	* @remarks No-long-jump zone!!!
496	*/
497	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
498	{
499	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
500	AssertRCReturn(rc, rc);
501	return VINF_SUCCESS;
502	}
503
504
505	/**
506	* Reads the VM-entry exception error code field from the VMCS into
507	* the VMX transient structure.
508	*
509	* @returns VBox status code.
510	* @param pVCpu Pointer to the VMCPU.
511	* @param pVmxTransient Pointer to the VMX transient structure.
512	*
513	* @remarks No-long-jump zone!!!
514	*/
515	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
516	{
517	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
518	AssertRCReturn(rc, rc);
519	return VINF_SUCCESS;
520	}
521
522
523	/**
524	* Reads the VM-exit interruption-information field from the VMCS into the VMX
525	* transient structure.
526	*
527	* @returns VBox status code.
528	* @param pVCpu Pointer to the VMCPU.
529	* @param pVmxTransient Pointer to the VMX transient structure.
530	*/
531	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
532	{
533	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
534	{
535	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
536	AssertRCReturn(rc, rc);
537	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
538	}
539	return VINF_SUCCESS;
540	}
541
542
543	/**
544	* Reads the VM-exit interruption error code from the VMCS into the VMX
545	* transient structure.
546	*
547	* @returns VBox status code.
548	* @param pVCpu Pointer to the VMCPU.
549	* @param pVmxTransient Pointer to the VMX transient structure.
550	*/
551	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
552	{
553	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
554	{
555	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
556	AssertRCReturn(rc, rc);
557	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
558	}
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Reads the VM-exit instruction length field from the VMCS into the VMX
565	* transient structure.
566	*
567	* @returns VBox status code.
568	* @param pVCpu Pointer to the VMCPU.
569	* @param pVmxTransient Pointer to the VMX transient structure.
570	*/
571	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
572	{
573	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
574	{
575	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
576	AssertRCReturn(rc, rc);
577	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
578	}
579	return VINF_SUCCESS;
580	}
581
582
583	/**
584	* Reads the exit qualification from the VMCS into the VMX transient structure.
585	*
586	* @returns VBox status code.
587	* @param pVCpu Pointer to the VMCPU.
588	* @param pVmxTransient Pointer to the VMX transient structure.
589	*/
590	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
591	{
592	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
593	{
594	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
595	AssertRCReturn(rc, rc);
596	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
597	}
598	return VINF_SUCCESS;
599	}
600
601
602	/**
603	* Reads the IDT-vectoring information field from the VMCS into the VMX
604	* transient structure.
605	*
606	* @returns VBox status code.
607	* @param pVmxTransient Pointer to the VMX transient structure.
608	*
609	* @remarks No-long-jump zone!!!
610	*/
611	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
612	{
613	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
614	{
615	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
616	AssertRCReturn(rc, rc);
617	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
618	}
619	return VINF_SUCCESS;
620	}
621
622
623	/**
624	* Reads the IDT-vectoring error code from the VMCS into the VMX
625	* transient structure.
626	*
627	* @returns VBox status code.
628	* @param pVmxTransient Pointer to the VMX transient structure.
629	*/
630	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
631	{
632	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
633	{
634	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
635	AssertRCReturn(rc, rc);
636	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
637	}
638	return VINF_SUCCESS;
639	}
640
641
642	/**
643	* Enters VMX root mode operation on the current CPU.
644	*
645	* @returns VBox status code.
646	* @param pVM Pointer to the VM (optional, can be NULL, after
647	* a resume).
648	* @param HCPhysCpuPage Physical address of the VMXON region.
649	* @param pvCpuPage Pointer to the VMXON region.
650	*/
651	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
652	{
653	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
654	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
655	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
656
657	if (pVM)
658	{
659	/* Write the VMCS revision dword to the VMXON region. */
660	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
661	}
662
663	/* Enable the VMX bit in CR4 if necessary. */
664	RTCCUINTREG uCr4 = ASMGetCR4();
665	if (!(uCr4 & X86_CR4_VMXE))
666	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
667
668	/* Enter VMX root mode. */
669	int rc = VMXEnable(HCPhysCpuPage);
670	if (RT_FAILURE(rc))
671	ASMSetCR4(uCr4);
672
673	return rc;
674	}
675
676
677	/**
678	* Exits VMX root mode operation on the current CPU.
679	*
680	* @returns VBox status code.
681	*/
682	static int hmR0VmxLeaveRootMode(void)
683	{
684	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
685
686	/* If we're for some reason not in VMX root mode, then don't leave it. */
687	RTCCUINTREG uHostCR4 = ASMGetCR4();
688	if (uHostCR4 & X86_CR4_VMXE)
689	{
690	/* Exit VMX root mode and clear the VMX bit in CR4. */
691	VMXDisable();
692	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
693	return VINF_SUCCESS;
694	}
695
696	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
697	}
698
699
700	/**
701	* Allocates and maps one physically contiguous page. The allocated page is
702	* zero'd out. (Used by various VT-x structures).
703	*
704	* @returns IPRT status code.
705	* @param pMemObj Pointer to the ring-0 memory object.
706	* @param ppVirt Where to store the virtual address of the
707	* allocation.
708	* @param pPhys Where to store the physical address of the
709	* allocation.
710	*/
711	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
712	{
713	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
714	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
715	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
716
717	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
718	if (RT_FAILURE(rc))
719	return rc;
720	ppVirt = RTR0MemObjAddress(pMemObj);
721	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
722	ASMMemZero32(*ppVirt, PAGE_SIZE);
723	return VINF_SUCCESS;
724	}
725
726
727	/**
728	* Frees and unmaps an allocated physical page.
729	*
730	* @param pMemObj Pointer to the ring-0 memory object.
731	* @param ppVirt Where to re-initialize the virtual address of
732	* allocation as 0.
733	* @param pHCPhys Where to re-initialize the physical address of the
734	* allocation as 0.
735	*/
736	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
737	{
738	AssertPtr(pMemObj);
739	AssertPtr(ppVirt);
740	AssertPtr(pHCPhys);
741	if (*pMemObj != NIL_RTR0MEMOBJ)
742	{
743	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
744	AssertRC(rc);
745	*pMemObj = NIL_RTR0MEMOBJ;
746	*ppVirt = 0;
747	*pHCPhys = 0;
748	}
749	}
750
751
752	/**
753	* Worker function to free VT-x related structures.
754	*
755	* @returns IPRT status code.
756	* @param pVM Pointer to the VM.
757	*/
758	static void hmR0VmxStructsFree(PVM pVM)
759	{
760	for (VMCPUID i = 0; i < pVM->cCpus; i++)
761	{
762	PVMCPU pVCpu = &pVM->aCpus[i];
763	AssertPtr(pVCpu);
764
765	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
766	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
767	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
768	#endif
769
770	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
771	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
772
773	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
774	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
775	}
776
777	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
778	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
779	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
780	#endif
781	}
782
783
784	/**
785	* Worker function to allocate VT-x related VM structures.
786	*
787	* @returns IPRT status code.
788	* @param pVM Pointer to the VM.
789	*/
790	static int hmR0VmxStructsAlloc(PVM pVM)
791	{
792	/*
793	* Initialize members up-front so we can cleanup properly on allocation failure.
794	*/
795	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
796	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
797	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
798	pVM->hm.s.vmx.HCPhys##a_Name = 0;
799
800	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
801	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
802	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
803	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
804
805	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
806	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
807	#endif
808	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
809
810	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
811	for (VMCPUID i = 0; i < pVM->cCpus; i++)
812	{
813	PVMCPU pVCpu = &pVM->aCpus[i];
814	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
815	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
816	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
817	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
818	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
819	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
820	#endif
821	}
822	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
823	#undef VMXLOCAL_INIT_VM_MEMOBJ
824
825	/*
826	* Allocate all the VT-x structures.
827	*/
828	int rc = VINF_SUCCESS;
829	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
830	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
831	if (RT_FAILURE(rc))
832	goto cleanup;
833	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
834	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
835	#endif
836
837	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
838	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
839	{
840	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
841	&pVM->hm.s.vmx.HCPhysApicAccess);
842	if (RT_FAILURE(rc))
843	goto cleanup;
844	}
845
846	/*
847	* Initialize per-VCPU VT-x structures.
848	*/
849	for (VMCPUID i = 0; i < pVM->cCpus; i++)
850	{
851	PVMCPU pVCpu = &pVM->aCpus[i];
852	AssertPtr(pVCpu);
853
854	/* Allocate the VM control structure (VMCS). */
855	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
856	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
857	if (RT_FAILURE(rc))
858	goto cleanup;
859
860	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
861	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
862	{
863	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
864	&pVCpu->hm.s.vmx.HCPhysVirtApic);
865	if (RT_FAILURE(rc))
866	goto cleanup;
867	}
868
869	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
870	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
871	{
872	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
873	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
874	if (RT_FAILURE(rc))
875	goto cleanup;
876	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
877	}
878
879	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
880	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
881	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
882	if (RT_FAILURE(rc))
883	goto cleanup;
884
885	/* Allocate the VM-exit MSR-load page for the host MSRs. */
886	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
887	if (RT_FAILURE(rc))
888	goto cleanup;
889	#endif
890	}
891
892	return VINF_SUCCESS;
893
894	cleanup:
895	hmR0VmxStructsFree(pVM);
896	return rc;
897	}
898
899
900	/**
901	* Does global VT-x initialization (called during module initialization).
902	*
903	* @returns VBox status code.
904	*/
905	VMMR0DECL(int) VMXR0GlobalInit(void)
906	{
907	#ifdef HMVMX_USE_FUNCTION_TABLE
908	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
909	# ifdef VBOX_STRICT
910	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
911	Assert(g_apfnVMExitHandlers[i]);
912	# endif
913	#endif
914	return VINF_SUCCESS;
915	}
916
917
918	/**
919	* Does global VT-x termination (called during module termination).
920	*/
921	VMMR0DECL(void) VMXR0GlobalTerm()
922	{
923	/* Nothing to do currently. */
924	}
925
926
927	/**
928	* Sets up and activates VT-x on the current CPU.
929	*
930	* @returns VBox status code.
931	* @param pCpu Pointer to the global CPU info struct.
932	* @param pVM Pointer to the VM (can be NULL after a host resume
933	* operation).
934	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
935	* fEnabledByHost is true).
936	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
937	* @a fEnabledByHost is true).
938	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
939	* enable VT-x on the host.
940	*/
941	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
942	{
943	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
944	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
945
946	if (!fEnabledByHost)
947	{
948	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
949	if (RT_FAILURE(rc))
950	return rc;
951	}
952
953	/*
954	* Flush all EPTP tagged-TLB entries (in case any other hypervisor have been using EPTPs) so that
955	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
956	* each time while reusing a VPID after hitting the MaxASID limit once.
957	*/
958	if ( pVM
959	&& pVM->hm.s.fNestedPaging)
960	{
961	/* We require ALL_CONTEXT flush-type to be available on the CPU. See hmR0VmxSetupTaggedTlb(). */
962	Assert(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
963	hmR0VmxFlushEpt(pVM, NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
964	pCpu->fFlushAsidBeforeUse = false;
965	}
966	else
967	{
968	/** @todo This is still not perfect. If on host resume (pVM is NULL or a VM
969	* without Nested Paging triggered this function) we still have the risk
970	* of potentially running with stale TLB-entries from other hypervisors
971	* when later we use a VM with NestedPaging. To fix this properly we will
972	* have to pass '&g_HvmR0' (see HMR0.cpp) to this function and read
973	* 'vmx_ept_vpid_caps' from it. Sigh. */
974	pCpu->fFlushAsidBeforeUse = true;
975	}
976
977	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
978	++pCpu->cTlbFlushes;
979
980	return VINF_SUCCESS;
981	}
982
983
984	/**
985	* Deactivates VT-x on the current CPU.
986	*
987	* @returns VBox status code.
988	* @param pCpu Pointer to the global CPU info struct.
989	* @param pvCpuPage Pointer to the VMXON region.
990	* @param HCPhysCpuPage Physical address of the VMXON region.
991	*
992	* @remarks This function should never be called when SUPR0EnableVTx() or
993	* similar was used to enable VT-x on the host.
994	*/
995	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
996	{
997	NOREF(pCpu);
998	NOREF(pvCpuPage);
999	NOREF(HCPhysCpuPage);
1000
1001	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1002	return hmR0VmxLeaveRootMode();
1003	}
1004
1005
1006	/**
1007	* Sets the permission bits for the specified MSR in the MSR bitmap.
1008	*
1009	* @param pVCpu Pointer to the VMCPU.
1010	* @param uMSR The MSR value.
1011	* @param enmRead Whether reading this MSR causes a VM-exit.
1012	* @param enmWrite Whether writing this MSR causes a VM-exit.
1013	*/
1014	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1015	{
1016	int32_t iBit;
1017	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1018
1019	/*
1020	* Layout:
1021	* 0x000 - 0x3ff - Low MSR read bits
1022	* 0x400 - 0x7ff - High MSR read bits
1023	* 0x800 - 0xbff - Low MSR write bits
1024	* 0xc00 - 0xfff - High MSR write bits
1025	*/
1026	if (uMsr <= 0x00001FFF)
1027	iBit = uMsr;
1028	else if ( uMsr >= 0xC0000000
1029	&& uMsr <= 0xC0001FFF)
1030	{
1031	iBit = (uMsr - 0xC0000000);
1032	pbMsrBitmap += 0x400;
1033	}
1034	else
1035	{
1036	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1037	return;
1038	}
1039
1040	Assert(iBit <= 0x1fff);
1041	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1042	ASMBitSet(pbMsrBitmap, iBit);
1043	else
1044	ASMBitClear(pbMsrBitmap, iBit);
1045
1046	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1047	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1048	else
1049	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1050	}
1051
1052
1053	/**
1054	* Flushes the TLB using EPT.
1055	*
1056	* @returns VBox status code.
1057	* @param pVM Pointer to the VM.
1058	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1059	* enmFlush).
1060	* @param enmFlush Type of flush.
1061	*/
1062	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1063	{
1064	AssertPtr(pVM);
1065	Assert(pVM->hm.s.fNestedPaging);
1066
1067	uint64_t descriptor[2];
1068	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1069	descriptor[0] = 0;
1070	else
1071	{
1072	Assert(pVCpu);
1073	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1074	}
1075	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1076
1077	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1078	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1079	rc));
1080	if ( RT_SUCCESS(rc)
1081	&& pVCpu)
1082	{
1083	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1084	}
1085	}
1086
1087
1088	/**
1089	* Flushes the TLB using VPID.
1090	*
1091	* @returns VBox status code.
1092	* @param pVM Pointer to the VM.
1093	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1094	* enmFlush).
1095	* @param enmFlush Type of flush.
1096	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1097	* on @a enmFlush).
1098	*/
1099	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1100	{
1101	AssertPtr(pVM);
1102	Assert(pVM->hm.s.vmx.fVpid);
1103
1104	uint64_t descriptor[2];
1105	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1106	{
1107	descriptor[0] = 0;
1108	descriptor[1] = 0;
1109	}
1110	else
1111	{
1112	AssertPtr(pVCpu);
1113	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1114	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1115	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1116	descriptor[1] = GCPtr;
1117	}
1118
1119	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1120	AssertMsg(rc == VINF_SUCCESS,
1121	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1122	if ( RT_SUCCESS(rc)
1123	&& pVCpu)
1124	{
1125	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1126	}
1127	}
1128
1129
1130	/**
1131	* Invalidates a guest page by guest virtual address. Only relevant for
1132	* EPT/VPID, otherwise there is nothing really to invalidate.
1133	*
1134	* @returns VBox status code.
1135	* @param pVM Pointer to the VM.
1136	* @param pVCpu Pointer to the VMCPU.
1137	* @param GCVirt Guest virtual address of the page to invalidate.
1138	*/
1139	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1140	{
1141	AssertPtr(pVM);
1142	AssertPtr(pVCpu);
1143	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1144
1145	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1146	if (!fFlushPending)
1147	{
1148	/*
1149	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1150	* See @bugref{6043} and @bugref{6177}.
1151	*
1152	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1153	* function maybe called in a loop with individual addresses.
1154	*/
1155	if (pVM->hm.s.vmx.fVpid)
1156	{
1157	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1158	{
1159	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1160	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1161	}
1162	else
1163	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1164	}
1165	else if (pVM->hm.s.fNestedPaging)
1166	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1167	}
1168
1169	return VINF_SUCCESS;
1170	}
1171
1172
1173	/**
1174	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1175	* otherwise there is nothing really to invalidate.
1176	*
1177	* @returns VBox status code.
1178	* @param pVM Pointer to the VM.
1179	* @param pVCpu Pointer to the VMCPU.
1180	* @param GCPhys Guest physical address of the page to invalidate.
1181	*/
1182	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1183	{
1184	LogFlowFunc(("%RGp\n", GCPhys));
1185
1186	/*
1187	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1188	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1189	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1190	*/
1191	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1192	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1193	return VINF_SUCCESS;
1194	}
1195
1196
1197	/**
1198	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1199	* case where neither EPT nor VPID is supported by the CPU.
1200	*
1201	* @param pVM Pointer to the VM.
1202	* @param pVCpu Pointer to the VMCPU.
1203	*
1204	* @remarks Called with interrupts disabled.
1205	*/
1206	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1207	{
1208	NOREF(pVM);
1209	AssertPtr(pVCpu);
1210	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1211	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1212
1213	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1214	AssertPtr(pCpu);
1215
1216	pVCpu->hm.s.TlbShootdown.cPages = 0;
1217	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1218	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1219	pVCpu->hm.s.fForceTLBFlush = false;
1220	return;
1221	}
1222
1223
1224	/**
1225	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1226	*
1227	* @param pVM Pointer to the VM.
1228	* @param pVCpu Pointer to the VMCPU.
1229	* @remarks All references to "ASID" in this function pertains to "VPID" in
1230	* Intel's nomenclature. The reason is, to avoid confusion in compare
1231	* statements since the host-CPU copies are named "ASID".
1232	*
1233	* @remarks Called with interrupts disabled.
1234	*/
1235	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1236	{
1237	#ifdef VBOX_WITH_STATISTICS
1238	bool fTlbFlushed = false;
1239	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1240	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1241	if (!fTlbFlushed) \
1242	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1243	} while (0)
1244	#else
1245	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1246	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1247	#endif
1248
1249	AssertPtr(pVM);
1250	AssertPtr(pVCpu);
1251	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1252	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1253	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1254
1255	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1256	AssertPtr(pCpu);
1257
1258	/*
1259	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1260	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1261	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1262	*/
1263	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1264	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1265	{
1266	++pCpu->uCurrentAsid;
1267	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1268	{
1269	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1270	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1271	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1272	}
1273
1274	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1275	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1276	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1277
1278	/*
1279	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1280	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1281	*/
1282	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1283	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1284	HMVMX_SET_TAGGED_TLB_FLUSHED();
1285	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1286	}
1287
1288	/* Check for explicit TLB shootdowns. */
1289	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1290	{
1291	/*
1292	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1293	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1294	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1295	* but not guest-physical mappings.
1296	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1297	*/
1298	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1299	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1300	HMVMX_SET_TAGGED_TLB_FLUSHED();
1301	}
1302
1303	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1304	* not be executed. See hmQueueInvlPage() where it is commented
1305	* out. Support individual entry flushing someday. */
1306	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1307	{
1308	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1309
1310	/*
1311	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1312	* as supported by the CPU.
1313	*/
1314	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1315	{
1316	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1317	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1318	}
1319	else
1320	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1321
1322	HMVMX_SET_TAGGED_TLB_FLUSHED();
1323	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1324	}
1325
1326	pVCpu->hm.s.TlbShootdown.cPages = 0;
1327	pVCpu->hm.s.fForceTLBFlush = false;
1328
1329	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1330
1331	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1332	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1333	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1334	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1335	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1336	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1337	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1338	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1339
1340	/* Update VMCS with the VPID. */
1341	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1342	AssertRC(rc);
1343
1344	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1345	}
1346
1347
1348	/**
1349	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1350	*
1351	* @returns VBox status code.
1352	* @param pVM Pointer to the VM.
1353	* @param pVCpu Pointer to the VMCPU.
1354	*
1355	* @remarks Called with interrupts disabled.
1356	*/
1357	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1358	{
1359	AssertPtr(pVM);
1360	AssertPtr(pVCpu);
1361	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1362	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1363
1364	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1365	AssertPtr(pCpu);
1366
1367	/*
1368	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1369	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1370	*/
1371	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1372	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1373	{
1374	pVCpu->hm.s.fForceTLBFlush = true;
1375	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1376	}
1377
1378	/* Check for explicit TLB shootdown flushes. */
1379	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1380	{
1381	pVCpu->hm.s.fForceTLBFlush = true;
1382	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1383	}
1384
1385	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1386	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1387
1388	if (pVCpu->hm.s.fForceTLBFlush)
1389	{
1390	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1391	pVCpu->hm.s.fForceTLBFlush = false;
1392	}
1393	else
1394	{
1395	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1396	* not be executed. See hmQueueInvlPage() where it is commented
1397	* out. Support individual entry flushing someday. */
1398	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1399	{
1400	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1401	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1402	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1403	}
1404	else
1405	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1406	}
1407
1408	pVCpu->hm.s.TlbShootdown.cPages = 0;
1409	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1410	}
1411
1412
1413	/**
1414	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1415	*
1416	* @returns VBox status code.
1417	* @param pVM Pointer to the VM.
1418	* @param pVCpu Pointer to the VMCPU.
1419	*
1420	* @remarks Called with interrupts disabled.
1421	*/
1422	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1423	{
1424	AssertPtr(pVM);
1425	AssertPtr(pVCpu);
1426	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1427	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1428
1429	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1430
1431	/*
1432	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1433	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1434	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1435	*/
1436	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1437	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1438	{
1439	pVCpu->hm.s.fForceTLBFlush = true;
1440	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1441	}
1442
1443	/* Check for explicit TLB shootdown flushes. */
1444	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1445	{
1446	/*
1447	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1448	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1449	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1450	*/
1451	pVCpu->hm.s.fForceTLBFlush = true;
1452	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1453	}
1454
1455	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1456	if (pVCpu->hm.s.fForceTLBFlush)
1457	{
1458	++pCpu->uCurrentAsid;
1459	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1460	{
1461	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1462	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1463	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1464	}
1465
1466	pVCpu->hm.s.fForceTLBFlush = false;
1467	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1468	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1469	if (pCpu->fFlushAsidBeforeUse)
1470	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1471	}
1472	else
1473	{
1474	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1475	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1476	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1477	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1478
1479	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1480	* not be executed. See hmQueueInvlPage() where it is commented
1481	* out. Support individual entry flushing someday. */
1482	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1483	{
1484	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1485	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1486	{
1487	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1488	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1489	}
1490	else
1491	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1492	}
1493	else
1494	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1495	}
1496
1497	pVCpu->hm.s.TlbShootdown.cPages = 0;
1498	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1499
1500	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1501	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1502	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1503	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1504	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1505	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1506
1507	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1508	AssertRC(rc);
1509	}
1510
1511
1512	/**
1513	* Flushes the guest TLB entry based on CPU capabilities.
1514	*
1515	* @param pVCpu Pointer to the VMCPU.
1516	*/
1517	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1518	{
1519	PVM pVM = pVCpu->CTX_SUFF(pVM);
1520	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1521	{
1522	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1523	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1524	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1525	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1526	default:
1527	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1528	break;
1529	}
1530	}
1531
1532
1533	/**
1534	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1535	* TLB entries from the host TLB before VM-entry.
1536	*
1537	* @returns VBox status code.
1538	* @param pVM Pointer to the VM.
1539	*/
1540	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1541	{
1542	/*
1543	* Determine optimal flush type for Nested Paging.
1544	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1545	* guest execution (see hmR3InitFinalizeR0()).
1546	*/
1547	if (pVM->hm.s.fNestedPaging)
1548	{
1549	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1550	{
1551	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1552	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1553	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1554	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1555	else
1556	{
1557	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1558	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1559	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1560	}
1561
1562	/* Make sure the write-back cacheable memory type for EPT is supported. */
1563	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1564	{
1565	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1566	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1567	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1568	}
1569	}
1570	else
1571	{
1572	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1573	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1574	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1575	}
1576	}
1577
1578	/*
1579	* Determine optimal flush type for VPID.
1580	*/
1581	if (pVM->hm.s.vmx.fVpid)
1582	{
1583	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1584	{
1585	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1586	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1587	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1588	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1589	else
1590	{
1591	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1592	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1593	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1594	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1595	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1596	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1597	pVM->hm.s.vmx.fVpid = false;
1598	}
1599	}
1600	else
1601	{
1602	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1603	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1604	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1605	pVM->hm.s.vmx.fVpid = false;
1606	}
1607	}
1608
1609	/*
1610	* Setup the handler for flushing tagged-TLBs.
1611	*/
1612	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1613	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1614	else if (pVM->hm.s.fNestedPaging)
1615	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1616	else if (pVM->hm.s.vmx.fVpid)
1617	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1618	else
1619	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1620	return VINF_SUCCESS;
1621	}
1622
1623
1624	/**
1625	* Sets up pin-based VM-execution controls in the VMCS.
1626	*
1627	* @returns VBox status code.
1628	* @param pVM Pointer to the VM.
1629	* @param pVCpu Pointer to the VMCPU.
1630	*/
1631	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1632	{
1633	AssertPtr(pVM);
1634	AssertPtr(pVCpu);
1635
1636	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1637	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1638
1639	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1640	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1641	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1642
1643	/* Enable the VMX preemption timer. */
1644	if (pVM->hm.s.vmx.fUsePreemptTimer)
1645	{
1646	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1647	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1648	}
1649
1650	if ((val & zap) != val)
1651	{
1652	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1653	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1654	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1655	}
1656
1657	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1658	AssertRCReturn(rc, rc);
1659
1660	/* Update VCPU with the currently set pin-based VM-execution controls. */
1661	pVCpu->hm.s.vmx.u32PinCtls = val;
1662	return rc;
1663	}
1664
1665
1666	/**
1667	* Sets up processor-based VM-execution controls in the VMCS.
1668	*
1669	* @returns VBox status code.
1670	* @param pVM Pointer to the VM.
1671	* @param pVMCPU Pointer to the VMCPU.
1672	*/
1673	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1674	{
1675	AssertPtr(pVM);
1676	AssertPtr(pVCpu);
1677
1678	int rc = VERR_INTERNAL_ERROR_5;
1679	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1680	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1681
1682	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1683	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1684	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1685	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1686	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1687	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1688	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1689
1690	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1691	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1692	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1693	{
1694	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1695	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1696	}
1697
1698	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1699	if (!pVM->hm.s.fNestedPaging)
1700	{
1701	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1702	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1703	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1704	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1705	}
1706
1707	/* Use TPR shadowing if supported by the CPU. */
1708	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1709	{
1710	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1711	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1712	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1713	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1714	AssertRCReturn(rc, rc);
1715
1716	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1717	/* CR8 writes causes a VM-exit based on TPR threshold. */
1718	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1719	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1720	}
1721	else
1722	{
1723	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1724	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1725	}
1726
1727	/* Use MSR-bitmaps if supported by the CPU. */
1728	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1729	{
1730	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1731
1732	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1733	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1734	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1735	AssertRCReturn(rc, rc);
1736
1737	/*
1738	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1739	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1740	*/
1741	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1742	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1743	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1744	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1745	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1746	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1747	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1748	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1749	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1750	}
1751
1752	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1753	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1754	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1755
1756	if ((val & zap) != val)
1757	{
1758	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1759	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1760	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1761	}
1762
1763	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1764	AssertRCReturn(rc, rc);
1765
1766	/* Update VCPU with the currently set processor-based VM-execution controls. */
1767	pVCpu->hm.s.vmx.u32ProcCtls = val;
1768
1769	/*
1770	* Secondary processor-based VM-execution controls.
1771	*/
1772	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1773	{
1774	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1775	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1776
1777	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1778	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1779
1780	if (pVM->hm.s.fNestedPaging)
1781	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1782	else
1783	{
1784	/*
1785	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1786	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1787	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1788	*/
1789	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1790	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1791	}
1792
1793	if (pVM->hm.s.vmx.fVpid)
1794	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1795
1796	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1797	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1798
1799	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1800	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1801	* done dynamically. */
1802	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1803	{
1804	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1805	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1806	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1807	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1808	AssertRCReturn(rc, rc);
1809	}
1810
1811	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1812	{
1813	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1814	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1815	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1816	}
1817
1818	if ((val & zap) != val)
1819	{
1820	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1821	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1822	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1823	}
1824
1825	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1826	AssertRCReturn(rc, rc);
1827
1828	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1829	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1830	}
1831
1832	return VINF_SUCCESS;
1833	}
1834
1835
1836	/**
1837	* Sets up miscellaneous (everything other than Pin & Processor-based
1838	* VM-execution) control fields in the VMCS.
1839	*
1840	* @returns VBox status code.
1841	* @param pVM Pointer to the VM.
1842	* @param pVCpu Pointer to the VMCPU.
1843	*/
1844	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1845	{
1846	AssertPtr(pVM);
1847	AssertPtr(pVCpu);
1848
1849	int rc = VERR_GENERAL_FAILURE;
1850
1851	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1852	#if 0
1853	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1854	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1855	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1856
1857	/*
1858	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1859	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1860	* We thus use the exception bitmap to control it rather than use both.
1861	*/
1862	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1863	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1864
1865	/** @todo Explore possibility of using IO-bitmaps. */
1866	/* All IO & IOIO instructions cause VM-exits. */
1867	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1868	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1869
1870	/* Initialize the MSR-bitmap area. */
1871	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1872	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1873	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1874	#endif
1875
1876	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1877	/* Setup MSR autoloading/storing. */
1878	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1879	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1880	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1881	AssertRCReturn(rc, rc);
1882	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1883	AssertRCReturn(rc, rc);
1884
1885	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1886	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1887	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1888	AssertRCReturn(rc, rc);
1889	#endif
1890
1891	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1892	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1893	AssertRCReturn(rc, rc);
1894
1895	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1896	#if 0
1897	/* Setup debug controls */
1898	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1899	AssertRCReturn(rc, rc);
1900	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1901	AssertRCReturn(rc, rc);
1902	#endif
1903
1904	return rc;
1905	}
1906
1907
1908	/**
1909	* Sets up the initial exception bitmap in the VMCS based on static conditions
1910	* (i.e. conditions that cannot ever change at runtime).
1911	*
1912	* @returns VBox status code.
1913	* @param pVM Pointer to the VM.
1914	* @param pVCpu Pointer to the VMCPU.
1915	*/
1916	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1917	{
1918	AssertPtr(pVM);
1919	AssertPtr(pVCpu);
1920
1921	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1922
1923	uint32_t u32XcptBitmap = 0;
1924
1925	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1926	if (!pVM->hm.s.fNestedPaging)
1927	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1928
1929	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1930	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1931	AssertRCReturn(rc, rc);
1932	return rc;
1933	}
1934
1935
1936	/**
1937	* Sets up the initial guest-state mask. The guest-state mask is consulted
1938	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1939	* for the nested virtualization case (as it would cause a VM-exit).
1940	*
1941	* @param pVCpu Pointer to the VMCPU.
1942	*/
1943	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1944	{
1945	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1946	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1947	return VINF_SUCCESS;
1948	}
1949
1950
1951	/**
1952	* Does per-VM VT-x initialization.
1953	*
1954	* @returns VBox status code.
1955	* @param pVM Pointer to the VM.
1956	*/
1957	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1958	{
1959	LogFlowFunc(("pVM=%p\n", pVM));
1960
1961	int rc = hmR0VmxStructsAlloc(pVM);
1962	if (RT_FAILURE(rc))
1963	{
1964	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1965	return rc;
1966	}
1967
1968	return VINF_SUCCESS;
1969	}
1970
1971
1972	/**
1973	* Does per-VM VT-x termination.
1974	*
1975	* @returns VBox status code.
1976	* @param pVM Pointer to the VM.
1977	*/
1978	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1979	{
1980	LogFlowFunc(("pVM=%p\n", pVM));
1981
1982	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1983	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1984	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1985	#endif
1986	hmR0VmxStructsFree(pVM);
1987	return VINF_SUCCESS;
1988	}
1989
1990
1991	/**
1992	* Sets up the VM for execution under VT-x.
1993	* This function is only called once per-VM during initalization.
1994	*
1995	* @returns VBox status code.
1996	* @param pVM Pointer to the VM.
1997	*/
1998	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1999	{
2000	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2001	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2002
2003	LogFlowFunc(("pVM=%p\n", pVM));
2004
2005	/*
2006	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2007	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2008	*/
2009	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2010	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2011	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2012	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2013	{
2014	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2015	return VERR_INTERNAL_ERROR;
2016	}
2017
2018	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2019	/*
2020	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2021	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2022	*/
2023	if ( pVM->hm.s.fAllow64BitGuests
2024	&& !HMVMX_IS_64BIT_HOST_MODE())
2025	{
2026	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2027	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2028	}
2029	#endif
2030
2031	/* Initialize these always, see hmR3InitFinalizeR0().*/
2032	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2033	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2034
2035	/* Setup the tagged-TLB flush handlers. */
2036	int rc = hmR0VmxSetupTaggedTlb(pVM);
2037	if (RT_FAILURE(rc))
2038	{
2039	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2040	return rc;
2041	}
2042
2043	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2044	{
2045	PVMCPU pVCpu = &pVM->aCpus[i];
2046	AssertPtr(pVCpu);
2047	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2048
2049	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2050	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2051
2052	/* Set revision dword at the beginning of the VMCS structure. */
2053	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
2054
2055	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2056	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2057	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2058	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2059
2060	/* Load this VMCS as the current VMCS. */
2061	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2062	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2063	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2064
2065	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2066	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2067	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2068
2069	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2070	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2071	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2072
2073	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2074	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2075	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2076
2077	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2078	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2079	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2080
2081	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2082	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2083	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2084
2085	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2086	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2087	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2088	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2089	#endif
2090
2091	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2092	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2093	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2094	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2095
2096	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2097	}
2098
2099	return VINF_SUCCESS;
2100	}
2101
2102
2103	/**
2104	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2105	* the VMCS.
2106	*
2107	* @returns VBox status code.
2108	* @param pVM Pointer to the VM.
2109	* @param pVCpu Pointer to the VMCPU.
2110	*/
2111	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2112	{
2113	RTCCUINTREG uReg = ASMGetCR0();
2114	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2115	AssertRCReturn(rc, rc);
2116
2117	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2118	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2119	if (HMVMX_IS_64BIT_HOST_MODE())
2120	{
2121	uint64_t uRegCR3 = HMR0Get64bitCR3();
2122	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2123	}
2124	else
2125	#endif
2126	{
2127	uReg = ASMGetCR3();
2128	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2129	}
2130	AssertRCReturn(rc, rc);
2131
2132	uReg = ASMGetCR4();
2133	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2134	AssertRCReturn(rc, rc);
2135	return rc;
2136	}
2137
2138
2139	/**
2140	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2141	* the host-state area in the VMCS.
2142	*
2143	* @returns VBox status code.
2144	* @param pVM Pointer to the VM.
2145	* @param pVCpu Pointer to the VMCPU.
2146	*/
2147	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2148	{
2149	int rc = VERR_INTERNAL_ERROR_5;
2150	RTSEL uSelDS = 0;
2151	RTSEL uSelES = 0;
2152	RTSEL uSelFS = 0;
2153	RTSEL uSelGS = 0;
2154	RTSEL uSelTR = 0;
2155
2156	/*
2157	* Host DS, ES, FS and GS segment registers.
2158	*/
2159	#if HC_ARCH_BITS == 64
2160	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2161	uSelDS = ASMGetDS();
2162	uSelES = ASMGetES();
2163	uSelFS = ASMGetFS();
2164	uSelGS = ASMGetGS();
2165	#endif
2166
2167	/*
2168	* Host CS and SS segment registers.
2169	*/
2170	RTSEL uSelCS;
2171	RTSEL uSelSS;
2172	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2173	if (HMVMX_IS_64BIT_HOST_MODE())
2174	{
2175	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2176	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2177	}
2178	else
2179	{
2180	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2181	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2182	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2183	}
2184	#else
2185	uSelCS = ASMGetCS();
2186	uSelSS = ASMGetSS();
2187	#endif
2188
2189	/*
2190	* Host TR segment register.
2191	*/
2192	uSelTR = ASMGetTR();
2193
2194	#if HC_ARCH_BITS == 64
2195	/*
2196	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2197	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2198	*/
2199	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2200	{
2201	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2202	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2203	uSelDS = 0;
2204	}
2205	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2206	{
2207	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2208	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2209	uSelES = 0;
2210	}
2211	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2212	{
2213	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2214	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2215	uSelFS = 0;
2216	}
2217	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2218	{
2219	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2220	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2221	uSelGS = 0;
2222	}
2223	#endif
2224
2225	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2226	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2227	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2228	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2229	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2230	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2231	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2232	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2233	Assert(uSelCS);
2234	Assert(uSelTR);
2235
2236	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2237	#if 0
2238	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2239	Assert(uSelSS != 0);
2240	#endif
2241
2242	/* Write these host selector fields into the host-state area in the VMCS. */
2243	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2244	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2245	#if HC_ARCH_BITS == 64
2246	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2247	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2248	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2249	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2250	#endif
2251	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2252
2253	/*
2254	* Host GDTR and IDTR.
2255	*/
2256	RTGDTR Gdtr;
2257	RT_ZERO(Gdtr);
2258	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2259	if (HMVMX_IS_64BIT_HOST_MODE())
2260	{
2261	X86XDTR64 Gdtr64;
2262	X86XDTR64 Idtr64;
2263	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2264	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2265	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2266
2267	Gdtr.cbGdt = Gdtr64.cb;
2268	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2269	}
2270	else
2271	#endif
2272	{
2273	RTIDTR Idtr;
2274	ASMGetGDTR(&Gdtr);
2275	ASMGetIDTR(&Idtr);
2276	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2277	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2278
2279	#if HC_ARCH_BITS == 64
2280	/*
2281	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2282	* maximum limit (0xffff) on every VM-exit.
2283	*/
2284	if (Gdtr.cbGdt != 0xffff)
2285	{
2286	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2287	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2288	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2289	}
2290
2291	/*
2292	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2293	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2294	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2295	*/
2296	if (Idtr.cbIdt < 0x0fff)
2297	{
2298	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2299	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2300	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2301	}
2302	#endif
2303	}
2304
2305	/*
2306	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2307	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2308	*/
2309	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2310	{
2311	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2312	return VERR_VMX_INVALID_HOST_STATE;
2313	}
2314
2315	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2316	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2317	if (HMVMX_IS_64BIT_HOST_MODE())
2318	{
2319	/* We need the 64-bit TR base for hybrid darwin. */
2320	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2321	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2322	}
2323	else
2324	#endif
2325	{
2326	uintptr_t uTRBase;
2327	#if HC_ARCH_BITS == 64
2328	uTRBase = X86DESC64_BASE(pDesc);
2329
2330	/*
2331	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2332	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2333	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2334	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2335	*
2336	* [1] See Intel spec. 3.5 "System Descriptor Types".
2337	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2338	*/
2339	Assert(pDesc->System.u4Type == 11);
2340	if ( pDesc->System.u16LimitLow != 0x67
2341	\|\| pDesc->System.u4LimitHigh)
2342	{
2343	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2344	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2345
2346	/* Store the GDTR here as we need it while restoring TR. */
2347	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2348	}
2349	#else
2350	uTRBase = X86DESC_BASE(pDesc);
2351	#endif
2352	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2353	}
2354	AssertRCReturn(rc, rc);
2355
2356	/*
2357	* Host FS base and GS base.
2358	*/
2359	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2360	if (HMVMX_IS_64BIT_HOST_MODE())
2361	{
2362	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2363	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2364	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2365	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2366
2367	# if HC_ARCH_BITS == 64
2368	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2369	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2370	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2371	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2372	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2373	# endif
2374	}
2375	#endif
2376	return rc;
2377	}
2378
2379
2380	/**
2381	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2382	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2383	* the host after every successful VM exit.
2384	*
2385	* @returns VBox status code.
2386	* @param pVM Pointer to the VM.
2387	* @param pVCpu Pointer to the VMCPU.
2388	*/
2389	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2390	{
2391	AssertPtr(pVCpu);
2392	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2393
2394	int rc = VINF_SUCCESS;
2395	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2396	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2397	uint32_t cHostMsrs = 0;
2398	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2399
2400	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2401	{
2402	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2403
2404	# if HC_ARCH_BITS == 64
2405	/* Paranoia. 64-bit code requires these bits to be set always. */
2406	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2407
2408	/*
2409	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2410	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2411	* some reason (e.g. allow transparent reads) we would activate the code below.
2412	*/
2413	# if 0
2414	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2415	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2416	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2417	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2418	if (CPUMIsGuestInLongMode(pVCpu))
2419	{
2420	uint64_t u64GuestEfer;
2421	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2422	AssertRC(rc);
2423
2424	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2425	{
2426	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2427	pHostMsr->u32Reserved = 0;
2428	pHostMsr->u64Value = u64HostEfer;
2429	pHostMsr++; cHostMsrs++;
2430	}
2431	}
2432	# endif
2433	# else /* HC_ARCH_BITS != 64 */
2434	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2435	pHostMsr->u32Reserved = 0;
2436	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2437	if (CPUMIsGuestInLongMode(pVCpu))
2438	{
2439	/* Must match the EFER value in our 64 bits switcher. */
2440	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2441	}
2442	else
2443	# endif
2444	pHostMsr->u64Value = u64HostEfer;
2445	pHostMsr++; cHostMsrs++;
2446	# endif /* HC_ARCH_BITS == 64 */
2447	}
2448
2449	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2450	if (HMVMX_IS_64BIT_HOST_MODE())
2451	{
2452	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2453	pHostMsr->u32Reserved = 0;
2454	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2455	pHostMsr++; cHostMsrs++;
2456	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2457	pHostMsr->u32Reserved = 0;
2458	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2459	pHostMsr++; cHostMsrs++;
2460	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2461	pHostMsr->u32Reserved = 0;
2462	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2463	pHostMsr++; cHostMsrs++;
2464	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2465	pHostMsr->u32Reserved = 0;
2466	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2467	pHostMsr++; cHostMsrs++;
2468	}
2469	# endif
2470
2471	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2472	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2473	{
2474	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2475	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2476	}
2477
2478	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2479	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2480
2481	/*
2482	* Host Sysenter MSRs.
2483	*/
2484	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2485	AssertRCReturn(rc, rc);
2486	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2487	if (HMVMX_IS_64BIT_HOST_MODE())
2488	{
2489	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2490	AssertRCReturn(rc, rc);
2491	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2492	}
2493	else
2494	{
2495	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2496	AssertRCReturn(rc, rc);
2497	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2498	}
2499	#elif HC_ARCH_BITS == 32
2500	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2501	AssertRCReturn(rc, rc);
2502	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2503	#else
2504	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2505	AssertRCReturn(rc, rc);
2506	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2507	#endif
2508	AssertRCReturn(rc, rc);
2509
2510	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2511	* hmR0VmxSetupExitCtls() !! */
2512	return rc;
2513	}
2514
2515
2516	/**
2517	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2518	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2519	* controls".
2520	*
2521	* @returns VBox status code.
2522	* @param pVCpu Pointer to the VMCPU.
2523	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2524	* out-of-sync. Make sure to update the required fields
2525	* before using them.
2526	*
2527	* @remarks No-long-jump zone!!!
2528	*/
2529	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2530	{
2531	int rc = VINF_SUCCESS;
2532	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2533	{
2534	PVM pVM = pVCpu->CTX_SUFF(pVM);
2535	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2536	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2537
2538	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2539	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2540
2541	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2542	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2543	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2544	else
2545	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2546
2547	/*
2548	* The following should not be set (since we're not in SMM mode):
2549	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2550	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2551	*/
2552
2553	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2554	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2555	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2556
2557	if ((val & zap) != val)
2558	{
2559	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2560	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2561	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2562	}
2563
2564	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2565	AssertRCReturn(rc, rc);
2566
2567	/* Update VCPU with the currently set VM-exit controls. */
2568	pVCpu->hm.s.vmx.u32EntryCtls = val;
2569	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2570	}
2571	return rc;
2572	}
2573
2574
2575	/**
2576	* Sets up the VM-exit controls in the VMCS.
2577	*
2578	* @returns VBox status code.
2579	* @param pVM Pointer to the VM.
2580	* @param pVCpu Pointer to the VMCPU.
2581	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2582	* out-of-sync. Make sure to update the required fields
2583	* before using them.
2584	*
2585	* @remarks requires EFER.
2586	*/
2587	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2588	{
2589	int rc = VINF_SUCCESS;
2590	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2591	{
2592	PVM pVM = pVCpu->CTX_SUFF(pVM);
2593	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2594	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2595
2596	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2597	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2598
2599	/*
2600	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2601	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2602	*/
2603	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2604	if (HMVMX_IS_64BIT_HOST_MODE())
2605	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2606	else
2607	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2608	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2609	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2610	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2611	else
2612	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2613	#endif
2614
2615	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2616	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2617
2618	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2619	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2620	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2621	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2622	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2623
2624	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2625	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2626
2627	if ((val & zap) != val)
2628	{
2629	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2630	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2631	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2632	}
2633
2634	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2635	AssertRCReturn(rc, rc);
2636
2637	/* Update VCPU with the currently set VM-exit controls. */
2638	pVCpu->hm.s.vmx.u32ExitCtls = val;
2639	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2640	}
2641	return rc;
2642	}
2643
2644
2645	/**
2646	* Loads the guest APIC and related state.
2647	*
2648	* @returns VBox status code.
2649	* @param pVM Pointer to the VM.
2650	* @param pVCpu Pointer to the VMCPU.
2651	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2652	* out-of-sync. Make sure to update the required fields
2653	* before using them.
2654	*/
2655	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2656	{
2657	int rc = VINF_SUCCESS;
2658	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2659	{
2660	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2661	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2662	{
2663	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2664
2665	bool fPendingIntr = false;
2666	uint8_t u8Tpr = 0;
2667	uint8_t u8PendingIntr = 0;
2668	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2669	AssertRCReturn(rc, rc);
2670
2671	/*
2672	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2673	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2674	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2675	* the interrupt when we VM-exit for other reasons.
2676	*/
2677	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2678	uint32_t u32TprThreshold = 0;
2679	if (fPendingIntr)
2680	{
2681	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2682	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2683	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2684	if (u8PendingPriority <= u8TprPriority)
2685	u32TprThreshold = u8PendingPriority;
2686	else
2687	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2688	}
2689	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2690
2691	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2692	AssertRCReturn(rc, rc);
2693	}
2694
2695	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2696	}
2697	return rc;
2698	}
2699
2700
2701	/**
2702	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2703	*
2704	* @returns Guest's interruptibility-state.
2705	* @param pVCpu Pointer to the VMCPU.
2706	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2707	* out-of-sync. Make sure to update the required fields
2708	* before using them.
2709	*
2710	* @remarks No-long-jump zone!!!
2711	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2712	*/
2713	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2714	{
2715	/*
2716	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2717	* inhibit interrupts or clear any existing interrupt-inhibition.
2718	*/
2719	uint32_t uIntrState = 0;
2720	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2721	{
2722	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2723	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2724	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2725	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2726	{
2727	/*
2728	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2729	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2730	*/
2731	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2732	}
2733	else if (pMixedCtx->eflags.Bits.u1IF)
2734	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2735	else
2736	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2737	}
2738	return uIntrState;
2739	}
2740
2741
2742	/**
2743	* Loads the guest's interruptibility-state into the guest-state area in the
2744	* VMCS.
2745	*
2746	* @returns VBox status code.
2747	* @param pVCpu Pointer to the VMCPU.
2748	* @param uIntrState The interruptibility-state to set.
2749	*/
2750	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2751	{
2752	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2753	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2754	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2755	AssertRCReturn(rc, rc);
2756	return rc;
2757	}
2758
2759
2760	/**
2761	* Loads the guest's RIP into the guest-state area in the VMCS.
2762	*
2763	* @returns VBox status code.
2764	* @param pVCpu Pointer to the VMCPU.
2765	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2766	* out-of-sync. Make sure to update the required fields
2767	* before using them.
2768	*
2769	* @remarks No-long-jump zone!!!
2770	*/
2771	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2772	{
2773	int rc = VINF_SUCCESS;
2774	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2775	{
2776	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2777	AssertRCReturn(rc, rc);
2778	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2779	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2780	}
2781	return rc;
2782	}
2783
2784
2785	/**
2786	* Loads the guest's RSP into the guest-state area in the VMCS.
2787	*
2788	* @returns VBox status code.
2789	* @param pVCpu Pointer to the VMCPU.
2790	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2791	* out-of-sync. Make sure to update the required fields
2792	* before using them.
2793	*
2794	* @remarks No-long-jump zone!!!
2795	*/
2796	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2797	{
2798	int rc = VINF_SUCCESS;
2799	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2800	{
2801	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2802	AssertRCReturn(rc, rc);
2803	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2804	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2805	}
2806	return rc;
2807	}
2808
2809
2810	/**
2811	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2812	*
2813	* @returns VBox status code.
2814	* @param pVCpu Pointer to the VMCPU.
2815	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2816	* out-of-sync. Make sure to update the required fields
2817	* before using them.
2818	*
2819	* @remarks No-long-jump zone!!!
2820	*/
2821	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2822	{
2823	int rc = VINF_SUCCESS;
2824	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2825	{
2826	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2827	Let us assert it as such and use 32-bit VMWRITE. */
2828	Assert(!(pMixedCtx->rflags.u64 >> 32));
2829	X86EFLAGS uEFlags = pMixedCtx->eflags;
2830	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2831	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2832
2833	/*
2834	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2835	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2836	*/
2837	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2838	{
2839	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2840	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2841	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2842	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2843	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2844	}
2845
2846	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2847	AssertRCReturn(rc, rc);
2848
2849	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2850	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2851	}
2852	return rc;
2853	}
2854
2855
2856	/**
2857	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2858	*
2859	* @returns VBox status code.
2860	* @param pVCpu Pointer to the VMCPU.
2861	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2862	* out-of-sync. Make sure to update the required fields
2863	* before using them.
2864	*
2865	* @remarks No-long-jump zone!!!
2866	*/
2867	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2868	{
2869	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2870	AssertRCReturn(rc, rc);
2871	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2872	AssertRCReturn(rc, rc);
2873	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2874	AssertRCReturn(rc, rc);
2875	return rc;
2876	}
2877
2878
2879	/**
2880	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2881	* in the VMCS.
2882	*
2883	* @returns VBox status code.
2884	* @param pVM Pointer to the VM.
2885	* @param pVCpu Pointer to the VMCPU.
2886	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2887	* out-of-sync. Make sure to update the required fields
2888	* before using them.
2889	*
2890	* @remarks No-long-jump zone!!!
2891	*/
2892	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2893	{
2894	int rc = VINF_SUCCESS;
2895	PVM pVM = pVCpu->CTX_SUFF(pVM);
2896
2897	/*
2898	* Guest CR0.
2899	* Guest FPU.
2900	*/
2901	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2902	{
2903	Assert(!(pCtx->cr0 >> 32));
2904	uint32_t u32GuestCR0 = pCtx->cr0;
2905
2906	/* The guest's view (read access) of its CR0 is unblemished. */
2907	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2908	AssertRCReturn(rc, rc);
2909	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2910
2911	/* Setup VT-x's view of the guest CR0. */
2912	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2913	if (pVM->hm.s.fNestedPaging)
2914	{
2915	if (CPUMIsGuestPagingEnabledEx(pCtx))
2916	{
2917	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2918	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2919	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2920	}
2921	else
2922	{
2923	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2924	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2925	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2926	}
2927
2928	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2929	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2930	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2931
2932	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2933	AssertRCReturn(rc, rc);
2934	}
2935	else
2936	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2937
2938	/*
2939	* Guest FPU bits.
2940	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2941	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2942	*/
2943	u32GuestCR0 \|= X86_CR0_NE;
2944	bool fInterceptNM = false;
2945	if (CPUMIsGuestFPUStateActive(pVCpu))
2946	{
2947	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2948	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2949	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2950	}
2951	else
2952	{
2953	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2954	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2955	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2956	}
2957
2958	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2959	bool fInterceptMF = false;
2960	if (!(pCtx->cr0 & X86_CR0_NE))
2961	fInterceptMF = true;
2962
2963	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2964	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2965	{
2966	Assert(PDMVmmDevHeapIsEnabled(pVM));
2967	Assert(pVM->hm.s.vmx.pRealModeTSS);
2968	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2969	fInterceptNM = true;
2970	fInterceptMF = true;
2971	}
2972	else
2973	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2974
2975	if (fInterceptNM)
2976	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2977	else
2978	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2979
2980	if (fInterceptMF)
2981	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2982	else
2983	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2984
2985	/* Additional intercepts for debugging, define these yourself explicitly. */
2986	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2987	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
2988	\| RT_BIT(X86_XCPT_BP)
2989	\| RT_BIT(X86_XCPT_DB)
2990	\| RT_BIT(X86_XCPT_DE)
2991	\| RT_BIT(X86_XCPT_NM)
2992	\| RT_BIT(X86_XCPT_UD)
2993	\| RT_BIT(X86_XCPT_NP)
2994	\| RT_BIT(X86_XCPT_SS)
2995	\| RT_BIT(X86_XCPT_GP)
2996	\| RT_BIT(X86_XCPT_PF)
2997	\| RT_BIT(X86_XCPT_MF)
2998	;
2999	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3000	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3001	#endif
3002
3003	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3004
3005	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3006	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3007	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3008	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3009	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3010	else
3011	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3012
3013	u32GuestCR0 \|= uSetCR0;
3014	u32GuestCR0 &= uZapCR0;
3015	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3016
3017	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3018	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3019	AssertRCReturn(rc, rc);
3020	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3021	AssertRCReturn(rc, rc);
3022	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3023
3024	/*
3025	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3026	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3027	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3028	*/
3029	uint32_t u32CR0Mask = 0;
3030	u32CR0Mask = X86_CR0_PE
3031	\| X86_CR0_NE
3032	\| X86_CR0_WP
3033	\| X86_CR0_PG
3034	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3035	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3036	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3037	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3038	u32CR0Mask &= ~X86_CR0_PE;
3039	if (pVM->hm.s.fNestedPaging)
3040	u32CR0Mask &= ~X86_CR0_WP;
3041
3042	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3043	if (fInterceptNM)
3044	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
3045	else
3046	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
3047
3048	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3049	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3050	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3051	AssertRCReturn(rc, rc);
3052
3053	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3054	}
3055
3056	/*
3057	* Guest CR2.
3058	* It's always loaded in the assembler code. Nothing to do here.
3059	*/
3060
3061	/*
3062	* Guest CR3.
3063	*/
3064	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3065	{
3066	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3067	if (pVM->hm.s.fNestedPaging)
3068	{
3069	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3070
3071	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3072	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3073	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3074	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3075
3076	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3077	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3078	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3079
3080	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3081	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3082	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3083	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3084
3085	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3086	AssertRCReturn(rc, rc);
3087	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3088
3089	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3090	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3091	{
3092	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3093	if (CPUMIsGuestInPAEModeEx(pCtx))
3094	{
3095	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3096	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3097	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3098	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3099	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3100	}
3101
3102	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3103	have Unrestricted Execution to handle the guest when it's not using paging. */
3104	GCPhysGuestCR3 = pCtx->cr3;
3105	}
3106	else
3107	{
3108	/*
3109	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3110	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3111	* EPT takes care of translating it to host-physical addresses.
3112	*/
3113	RTGCPHYS GCPhys;
3114	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3115	Assert(PDMVmmDevHeapIsEnabled(pVM));
3116
3117	/* We obtain it here every time as the guest could have relocated this PCI region. */
3118	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3119	AssertRCReturn(rc, rc);
3120
3121	GCPhysGuestCR3 = GCPhys;
3122	}
3123
3124	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3125	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3126	}
3127	else
3128	{
3129	/* Non-nested paging case, just use the hypervisor's CR3. */
3130	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3131
3132	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3133	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3134	}
3135	AssertRCReturn(rc, rc);
3136
3137	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3138	}
3139
3140	/*
3141	* Guest CR4.
3142	*/
3143	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3144	{
3145	Assert(!(pCtx->cr4 >> 32));
3146	uint32_t u32GuestCR4 = pCtx->cr4;
3147
3148	/* The guest's view of its CR4 is unblemished. */
3149	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3150	AssertRCReturn(rc, rc);
3151	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3152
3153	/* Setup VT-x's view of the guest CR4. */
3154	/*
3155	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3156	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3157	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3158	*/
3159	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3160	{
3161	Assert(pVM->hm.s.vmx.pRealModeTSS);
3162	Assert(PDMVmmDevHeapIsEnabled(pVM));
3163	u32GuestCR4 &= ~X86_CR4_VME;
3164	}
3165
3166	if (pVM->hm.s.fNestedPaging)
3167	{
3168	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3169	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3170	{
3171	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3172	u32GuestCR4 \|= X86_CR4_PSE;
3173	/* Our identity mapping is a 32 bits page directory. */
3174	u32GuestCR4 &= ~X86_CR4_PAE;
3175	}
3176	/* else use guest CR4.*/
3177	}
3178	else
3179	{
3180	/*
3181	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3182	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3183	*/
3184	switch (pVCpu->hm.s.enmShadowMode)
3185	{
3186	case PGMMODE_REAL: /* Real-mode. */
3187	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3188	case PGMMODE_32_BIT: /* 32-bit paging. */
3189	{
3190	u32GuestCR4 &= ~X86_CR4_PAE;
3191	break;
3192	}
3193
3194	case PGMMODE_PAE: /* PAE paging. */
3195	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3196	{
3197	u32GuestCR4 \|= X86_CR4_PAE;
3198	break;
3199	}
3200
3201	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3202	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3203	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3204	break;
3205	#endif
3206	default:
3207	AssertFailed();
3208	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3209	}
3210	}
3211
3212	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3213	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3214	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3215	u32GuestCR4 \|= uSetCR4;
3216	u32GuestCR4 &= uZapCR4;
3217
3218	/* Write VT-x's view of the guest CR4 into the VMCS. */
3219	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3220	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3221	AssertRCReturn(rc, rc);
3222
3223	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3224	uint32_t u32CR4Mask = 0;
3225	u32CR4Mask = X86_CR4_VME
3226	\| X86_CR4_PAE
3227	\| X86_CR4_PGE
3228	\| X86_CR4_PSE
3229	\| X86_CR4_VMXE;
3230	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3231	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3232	AssertRCReturn(rc, rc);
3233
3234	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3235	}
3236	return rc;
3237	}
3238
3239
3240	/**
3241	* Loads the guest debug registers into the guest-state area in the VMCS.
3242	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3243	*
3244	* @returns VBox status code.
3245	* @param pVCpu Pointer to the VMCPU.
3246	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3247	* out-of-sync. Make sure to update the required fields
3248	* before using them.
3249	*
3250	* @remarks No-long-jump zone!!!
3251	*/
3252	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3253	{
3254	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3255	return VINF_SUCCESS;
3256
3257	#ifdef VBOX_STRICT
3258	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3259	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3260	{
3261	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3262	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3263	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3264	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3265	}
3266	#endif
3267
3268	int rc = VERR_INTERNAL_ERROR_5;
3269	PVM pVM = pVCpu->CTX_SUFF(pVM);
3270	bool fInterceptDB = false;
3271	bool fInterceptMovDRx = false;
3272	if (DBGFIsStepping(pVCpu))
3273	{
3274	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3275	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3276	{
3277	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3278	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3279	AssertRCReturn(rc, rc);
3280	Assert(fInterceptDB == false);
3281	}
3282	else
3283	{
3284	fInterceptDB = true;
3285	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3286	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3287	}
3288	}
3289
3290	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3291	{
3292	if (!CPUMIsHyperDebugStateActive(pVCpu))
3293	{
3294	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3295	AssertRC(rc);
3296	}
3297	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3298	fInterceptMovDRx = true;
3299	}
3300	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3301	{
3302	if (!CPUMIsGuestDebugStateActive(pVCpu))
3303	{
3304	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3305	AssertRC(rc);
3306	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3307	}
3308	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3309	Assert(fInterceptMovDRx == false);
3310	}
3311	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3312	{
3313	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3314	fInterceptMovDRx = true;
3315	}
3316
3317	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3318	if (fInterceptDB)
3319	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3320	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3321	{
3322	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3323	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3324	#endif
3325	}
3326
3327	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3328	if (fInterceptMovDRx)
3329	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3330	else
3331	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3332
3333	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3334	AssertRCReturn(rc, rc);
3335	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3336	AssertRCReturn(rc, rc);
3337
3338	/* The guest's view of its DR7 is unblemished. Use 32-bit write as upper 32-bits MBZ as asserted above. */
3339	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
3340	AssertRCReturn(rc, rc);
3341
3342	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3343	return rc;
3344	}
3345
3346
3347	#ifdef VBOX_STRICT
3348	/**
3349	* Strict function to validate segment registers.
3350	*
3351	* @remarks Requires CR0.
3352	*/
3353	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3354	{
3355	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3356	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3357	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3358	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3359	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3360	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3361	{
3362	/* Protected mode checks */
3363	/* CS */
3364	Assert(pCtx->cs.Attr.n.u1Present);
3365	Assert(!(pCtx->cs.Attr.u & 0xf00));
3366	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3367	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3368	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3369	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3370	\|\| (pCtx->cs.Attr.n.u1Granularity));
3371	/* CS cannot be loaded with NULL in protected mode. */
3372	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3373	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3374	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3375	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3376	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3377	else
3378	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3379	/* SS */
3380	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3381	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3382	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3383	if ( !(pCtx->cr0 & X86_CR0_PE)
3384	\|\| pCtx->cs.Attr.n.u4Type == 3)
3385	{
3386	Assert(!pCtx->ss.Attr.n.u2Dpl);
3387	}
3388	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3389	{
3390	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3391	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3392	Assert(pCtx->ss.Attr.n.u1Present);
3393	Assert(!(pCtx->ss.Attr.u & 0xf00));
3394	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3395	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3396	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3397	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3398	\|\| (pCtx->ss.Attr.n.u1Granularity));
3399	}
3400	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3401	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3402	{
3403	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3404	Assert(pCtx->ds.Attr.n.u1Present);
3405	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3406	Assert(!(pCtx->ds.Attr.u & 0xf00));
3407	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3408	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3409	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3410	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3411	\|\| (pCtx->ds.Attr.n.u1Granularity));
3412	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3413	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3414	}
3415	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3416	{
3417	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3418	Assert(pCtx->es.Attr.n.u1Present);
3419	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3420	Assert(!(pCtx->es.Attr.u & 0xf00));
3421	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3422	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3423	\|\| !(pCtx->es.Attr.n.u1Granularity));
3424	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3425	\|\| (pCtx->es.Attr.n.u1Granularity));
3426	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3427	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3428	}
3429	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3430	{
3431	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3432	Assert(pCtx->fs.Attr.n.u1Present);
3433	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3434	Assert(!(pCtx->fs.Attr.u & 0xf00));
3435	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3436	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3437	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3438	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3439	\|\| (pCtx->fs.Attr.n.u1Granularity));
3440	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3441	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3442	}
3443	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3444	{
3445	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3446	Assert(pCtx->gs.Attr.n.u1Present);
3447	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3448	Assert(!(pCtx->gs.Attr.u & 0xf00));
3449	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3450	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3451	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3452	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3453	\|\| (pCtx->gs.Attr.n.u1Granularity));
3454	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3455	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3456	}
3457	/* 64-bit capable CPUs. */
3458	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3459	Assert(!(pCtx->cs.u64Base >> 32));
3460	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3461	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3462	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3463	# endif
3464	}
3465	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3466	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3467	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3468	{
3469	/* Real and v86 mode checks. */
3470	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3471	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3472	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3473	{
3474	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3475	}
3476	else
3477	{
3478	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3479	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3480	}
3481
3482	/* CS */
3483	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3484	Assert(pCtx->cs.u32Limit == 0xffff);
3485	Assert(u32CSAttr == 0xf3);
3486	/* SS */
3487	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3488	Assert(pCtx->ss.u32Limit == 0xffff);
3489	Assert(u32SSAttr == 0xf3);
3490	/* DS */
3491	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3492	Assert(pCtx->ds.u32Limit == 0xffff);
3493	Assert(u32DSAttr == 0xf3);
3494	/* ES */
3495	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3496	Assert(pCtx->es.u32Limit == 0xffff);
3497	Assert(u32ESAttr == 0xf3);
3498	/* FS */
3499	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3500	Assert(pCtx->fs.u32Limit == 0xffff);
3501	Assert(u32FSAttr == 0xf3);
3502	/* GS */
3503	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3504	Assert(pCtx->gs.u32Limit == 0xffff);
3505	Assert(u32GSAttr == 0xf3);
3506	/* 64-bit capable CPUs. */
3507	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3508	Assert(!(pCtx->cs.u64Base >> 32));
3509	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3510	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3511	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3512	# endif
3513	}
3514	}
3515	#endif /* VBOX_STRICT */
3516
3517
3518	/**
3519	* Writes a guest segment register into the guest-state area in the VMCS.
3520	*
3521	* @returns VBox status code.
3522	* @param pVCpu Pointer to the VMCPU.
3523	* @param idxSel Index of the selector in the VMCS.
3524	* @param idxLimit Index of the segment limit in the VMCS.
3525	* @param idxBase Index of the segment base in the VMCS.
3526	* @param idxAccess Index of the access rights of the segment in the VMCS.
3527	* @param pSelReg Pointer to the segment selector.
3528	* @param pCtx Pointer to the guest-CPU context.
3529	*
3530	* @remarks No-long-jump zone!!!
3531	*/
3532	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3533	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3534	{
3535	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3536	AssertRCReturn(rc, rc);
3537	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3538	AssertRCReturn(rc, rc);
3539	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3540	AssertRCReturn(rc, rc);
3541
3542	uint32_t u32Access = pSelReg->Attr.u;
3543	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3544	{
3545	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3546	u32Access = 0xf3;
3547	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3548	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3549	}
3550	else
3551	{
3552	/*
3553	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3554	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3555	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3556	* loaded in protected-mode have their attribute as 0.
3557	*/
3558	if (!u32Access)
3559	u32Access = X86DESCATTR_UNUSABLE;
3560	}
3561
3562	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3563	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3564	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3565
3566	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3567	AssertRCReturn(rc, rc);
3568	return rc;
3569	}
3570
3571
3572	/**
3573	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3574	* into the guest-state area in the VMCS.
3575	*
3576	* @returns VBox status code.
3577	* @param pVM Pointer to the VM.
3578	* @param pVCPU Pointer to the VMCPU.
3579	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3580	* out-of-sync. Make sure to update the required fields
3581	* before using them.
3582	*
3583	* @remarks Requires CR0 (strict builds validation).
3584	* @remarks No-long-jump zone!!!
3585	*/
3586	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3587	{
3588	int rc = VERR_INTERNAL_ERROR_5;
3589	PVM pVM = pVCpu->CTX_SUFF(pVM);
3590
3591	/*
3592	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3593	*/
3594	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3595	{
3596	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3597	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3598	{
3599	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3600	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3601	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3602	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3603	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3604	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3605	}
3606
3607	#ifdef VBOX_WITH_REM
3608	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3609	{
3610	Assert(pVM->hm.s.vmx.pRealModeTSS);
3611	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3612	if ( pVCpu->hm.s.vmx.fWasInRealMode
3613	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3614	{
3615	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3616	in real-mode (e.g. OpenBSD 4.0) */
3617	REMFlushTBs(pVM);
3618	Log4(("Load: Switch to protected mode detected!\n"));
3619	pVCpu->hm.s.vmx.fWasInRealMode = false;
3620	}
3621	}
3622	#endif
3623	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3624	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3625	AssertRCReturn(rc, rc);
3626	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3627	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3628	AssertRCReturn(rc, rc);
3629	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3630	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3631	AssertRCReturn(rc, rc);
3632	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3633	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3634	AssertRCReturn(rc, rc);
3635	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3636	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3637	AssertRCReturn(rc, rc);
3638	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3639	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3640	AssertRCReturn(rc, rc);
3641
3642	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3643	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3644	#ifdef VBOX_STRICT
3645	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3646	#endif
3647	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3648	}
3649
3650	/*
3651	* Guest TR.
3652	*/
3653	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3654	{
3655	/*
3656	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3657	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3658	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3659	*/
3660	uint16_t u16Sel = 0;
3661	uint32_t u32Limit = 0;
3662	uint64_t u64Base = 0;
3663	uint32_t u32AccessRights = 0;
3664
3665	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3666	{
3667	u16Sel = pMixedCtx->tr.Sel;
3668	u32Limit = pMixedCtx->tr.u32Limit;
3669	u64Base = pMixedCtx->tr.u64Base;
3670	u32AccessRights = pMixedCtx->tr.Attr.u;
3671	}
3672	else
3673	{
3674	Assert(pVM->hm.s.vmx.pRealModeTSS);
3675	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3676
3677	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3678	RTGCPHYS GCPhys;
3679	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3680	AssertRCReturn(rc, rc);
3681
3682	X86DESCATTR DescAttr;
3683	DescAttr.u = 0;
3684	DescAttr.n.u1Present = 1;
3685	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3686
3687	u16Sel = 0;
3688	u32Limit = HM_VTX_TSS_SIZE;
3689	u64Base = GCPhys; /* in real-mode phys = virt. */
3690	u32AccessRights = DescAttr.u;
3691	}
3692
3693	/* Validate. */
3694	Assert(!(u16Sel & RT_BIT(2)));
3695	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3696	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3697	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3698	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3699	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3700	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3701	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3702	Assert( (u32Limit & 0xfff) == 0xfff
3703	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3704	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3705	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3706
3707	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3708	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3709	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3710	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3711
3712	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3713	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3714	}
3715
3716	/*
3717	* Guest GDTR.
3718	*/
3719	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3720	{
3721	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3722	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3723
3724	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3725	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3726	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3727	}
3728
3729	/*
3730	* Guest LDTR.
3731	*/
3732	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3733	{
3734	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3735	uint32_t u32Access = 0;
3736	if (!pMixedCtx->ldtr.Attr.u)
3737	u32Access = X86DESCATTR_UNUSABLE;
3738	else
3739	u32Access = pMixedCtx->ldtr.Attr.u;
3740
3741	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3742	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3743	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3744	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3745
3746	/* Validate. */
3747	if (!(u32Access & X86DESCATTR_UNUSABLE))
3748	{
3749	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3750	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3751	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3752	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3753	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3754	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3755	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3756	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3757	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3758	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3759	}
3760
3761	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3762	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3763	}
3764
3765	/*
3766	* Guest IDTR.
3767	*/
3768	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3769	{
3770	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3771	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3772
3773	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3774	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3775	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3776	}
3777
3778	return VINF_SUCCESS;
3779	}
3780
3781
3782	/**
3783	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3784	* areas. These MSRs will automatically be loaded to the host CPU on every
3785	* successful VM entry and stored from the host CPU on every successful VM exit.
3786	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3787	*
3788	* @returns VBox status code.
3789	* @param pVCpu Pointer to the VMCPU.
3790	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3791	* out-of-sync. Make sure to update the required fields
3792	* before using them.
3793	*
3794	* @remarks No-long-jump zone!!!
3795	*/
3796	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3797	{
3798	AssertPtr(pVCpu);
3799	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3800
3801	/*
3802	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3803	*/
3804	int rc = VINF_SUCCESS;
3805	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3806	{
3807	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3808	PVM pVM = pVCpu->CTX_SUFF(pVM);
3809	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3810	uint32_t cGuestMsrs = 0;
3811
3812	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3813	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3814	* when the guest really is in 64-bit mode. */
3815	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3816	if (fSupportsLongMode)
3817	{
3818	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3819	pGuestMsr->u32Reserved = 0;
3820	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3821	pGuestMsr++; cGuestMsrs++;
3822	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3823	pGuestMsr->u32Reserved = 0;
3824	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3825	pGuestMsr++; cGuestMsrs++;
3826	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3827	pGuestMsr->u32Reserved = 0;
3828	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3829	pGuestMsr++; cGuestMsrs++;
3830	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3831	pGuestMsr->u32Reserved = 0;
3832	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3833	pGuestMsr++; cGuestMsrs++;
3834	}
3835
3836	/*
3837	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3838	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3839	*/
3840	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3841	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3842	{
3843	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3844	pGuestMsr->u32Reserved = 0;
3845	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3846	AssertRCReturn(rc, rc);
3847	pGuestMsr++; cGuestMsrs++;
3848	}
3849
3850	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3851	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3852	{
3853	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3854	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3855	}
3856
3857	/* Update the VCPU's copy of the guest MSR count. */
3858	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3859	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3860	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3861	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3862
3863	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3864	}
3865
3866	/*
3867	* Guest Sysenter MSRs.
3868	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3869	* VM-exits on WRMSRs for these MSRs.
3870	*/
3871	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3872	{
3873	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3874	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3875	}
3876	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3877	{
3878	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3879	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3880	}
3881	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3882	{
3883	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3884	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3885	}
3886
3887	return rc;
3888	}
3889
3890
3891	/**
3892	* Loads the guest activity state into the guest-state area in the VMCS.
3893	*
3894	* @returns VBox status code.
3895	* @param pVCpu Pointer to the VMCPU.
3896	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3897	* out-of-sync. Make sure to update the required fields
3898	* before using them.
3899	*
3900	* @remarks No-long-jump zone!!!
3901	*/
3902	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3903	{
3904	/** @todo See if we can make use of other states, e.g.
3905	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3906	int rc = VINF_SUCCESS;
3907	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3908	{
3909	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3910	AssertRCReturn(rc, rc);
3911	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3912	}
3913	return rc;
3914	}
3915
3916
3917	/**
3918	* Sets up the appropriate function to run guest code.
3919	*
3920	* @returns VBox status code.
3921	* @param pVCpu Pointer to the VMCPU.
3922	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3923	* out-of-sync. Make sure to update the required fields
3924	* before using them.
3925	*
3926	* @remarks No-long-jump zone!!!
3927	*/
3928	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3929	{
3930	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3931	{
3932	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3933	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3934	#endif
3935	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3936	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3937	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3938	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3939	#else
3940	/* 64-bit host or hybrid host. */
3941	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3942	#endif
3943	}
3944	else
3945	{
3946	/* Guest is not in long mode, use the 32-bit handler. */
3947	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3948	}
3949	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3950	return VINF_SUCCESS;
3951	}
3952
3953
3954	/**
3955	* Wrapper for running the guest code in VT-x.
3956	*
3957	* @returns VBox strict status code.
3958	* @param pVM Pointer to the VM.
3959	* @param pVCpu Pointer to the VMCPU.
3960	* @param pCtx Pointer to the guest-CPU context.
3961	*
3962	* @remarks No-long-jump zone!!!
3963	*/
3964	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3965	{
3966	/*
3967	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3968	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3969	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3970	*/
3971	#ifdef VBOX_WITH_KERNEL_USING_XMM
3972	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3973	#else
3974	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3975	#endif
3976	}
3977
3978
3979	/**
3980	* Reports world-switch error and dumps some useful debug info.
3981	*
3982	* @param pVM Pointer to the VM.
3983	* @param pVCpu Pointer to the VMCPU.
3984	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3985	* @param pCtx Pointer to the guest-CPU context.
3986	* @param pVmxTransient Pointer to the VMX transient structure (only
3987	* exitReason updated).
3988	*/
3989	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3990	{
3991	Assert(pVM);
3992	Assert(pVCpu);
3993	Assert(pCtx);
3994	Assert(pVmxTransient);
3995	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3996
3997	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
3998	switch (rcVMRun)
3999	{
4000	case VERR_VMX_INVALID_VMXON_PTR:
4001	AssertFailed();
4002	break;
4003	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4004	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4005	{
4006	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4007	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4008	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4009	AssertRC(rc);
4010
4011	#ifdef VBOX_STRICT
4012	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4013	pVmxTransient->uExitReason));
4014	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4015	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4016	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4017	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4018	else
4019	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4020
4021	/* VMX control bits. */
4022	uint32_t u32Val;
4023	uint64_t u64Val;
4024	HMVMXHCUINTREG uHCReg;
4025	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4026	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4027	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4028	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4029	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4030	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4031	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4032	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4033	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4034	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4035	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4036	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4037	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4038	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4039	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4040	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4041	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4042	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4043	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4044	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4045	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4046	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4047	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4048	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4049	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4050	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4051	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4052	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4053	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4054	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4055	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4056	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4057	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4058	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4059	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4060	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4061	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4062	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4063	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4064	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4065	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4066	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4067
4068	/* Guest bits. */
4069	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4070	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4071	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4072	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4073	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4074	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4075	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4076	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4077
4078	/* Host bits. */
4079	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4080	Log4(("Host CR0 %#RHr\n", uHCReg));
4081	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4082	Log4(("Host CR3 %#RHr\n", uHCReg));
4083	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4084	Log4(("Host CR4 %#RHr\n", uHCReg));
4085
4086	RTGDTR HostGdtr;
4087	PCX86DESCHC pDesc;
4088	ASMGetGDTR(&HostGdtr);
4089	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4090	Log4(("Host CS %#08x\n", u32Val));
4091	if (u32Val < HostGdtr.cbGdt)
4092	{
4093	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4094	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4095	}
4096
4097	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4098	Log4(("Host DS %#08x\n", u32Val));
4099	if (u32Val < HostGdtr.cbGdt)
4100	{
4101	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4102	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4103	}
4104
4105	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4106	Log4(("Host ES %#08x\n", u32Val));
4107	if (u32Val < HostGdtr.cbGdt)
4108	{
4109	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4110	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4111	}
4112
4113	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4114	Log4(("Host FS %#08x\n", u32Val));
4115	if (u32Val < HostGdtr.cbGdt)
4116	{
4117	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4118	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4119	}
4120
4121	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4122	Log4(("Host GS %#08x\n", u32Val));
4123	if (u32Val < HostGdtr.cbGdt)
4124	{
4125	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4126	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4127	}
4128
4129	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4130	Log4(("Host SS %#08x\n", u32Val));
4131	if (u32Val < HostGdtr.cbGdt)
4132	{
4133	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4134	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4135	}
4136
4137	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4138	Log4(("Host TR %#08x\n", u32Val));
4139	if (u32Val < HostGdtr.cbGdt)
4140	{
4141	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4142	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4143	}
4144
4145	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4146	Log4(("Host TR Base %#RHv\n", uHCReg));
4147	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4148	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4149	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4150	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4151	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4152	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4153	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4154	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4155	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4156	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4157	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4158	Log4(("Host RSP %#RHv\n", uHCReg));
4159	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4160	Log4(("Host RIP %#RHv\n", uHCReg));
4161	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4162	if (HMVMX_IS_64BIT_HOST_MODE())
4163	{
4164	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4165	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4166	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4167	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4168	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4169	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4170	}
4171	# endif
4172	#endif /* VBOX_STRICT */
4173	break;
4174	}
4175
4176	default:
4177	/* Impossible */
4178	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4179	break;
4180	}
4181	NOREF(pVM);
4182	}
4183
4184
4185	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4186	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4187	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4188	#endif
4189	#ifdef VBOX_STRICT
4190	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4191	{
4192	switch (idxField)
4193	{
4194	case VMX_VMCS_GUEST_RIP:
4195	case VMX_VMCS_GUEST_RSP:
4196	case VMX_VMCS_GUEST_SYSENTER_EIP:
4197	case VMX_VMCS_GUEST_SYSENTER_ESP:
4198	case VMX_VMCS_GUEST_GDTR_BASE:
4199	case VMX_VMCS_GUEST_IDTR_BASE:
4200	case VMX_VMCS_GUEST_CS_BASE:
4201	case VMX_VMCS_GUEST_DS_BASE:
4202	case VMX_VMCS_GUEST_ES_BASE:
4203	case VMX_VMCS_GUEST_FS_BASE:
4204	case VMX_VMCS_GUEST_GS_BASE:
4205	case VMX_VMCS_GUEST_SS_BASE:
4206	case VMX_VMCS_GUEST_LDTR_BASE:
4207	case VMX_VMCS_GUEST_TR_BASE:
4208	case VMX_VMCS_GUEST_CR3:
4209	return true;
4210	}
4211	return false;
4212	}
4213
4214	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4215	{
4216	switch (idxField)
4217	{
4218	/* Read-only fields. */
4219	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4220	return true;
4221	}
4222	/* Remaining readable fields should also be writable. */
4223	return hmR0VmxIsValidWriteField(idxField);
4224	}
4225	#endif /* VBOX_STRICT */
4226
4227
4228	/**
4229	* Executes the specified handler in 64-bit mode.
4230	*
4231	* @returns VBox status code.
4232	* @param pVM Pointer to the VM.
4233	* @param pVCpu Pointer to the VMCPU.
4234	* @param pCtx Pointer to the guest CPU context.
4235	* @param enmOp The operation to perform.
4236	* @param cbParam Number of parameters.
4237	* @param paParam Array of 32-bit parameters.
4238	*/
4239	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4240	uint32_t *paParam)
4241	{
4242	int rc, rc2;
4243	PHMGLOBLCPUINFO pCpu;
4244	RTHCPHYS HCPhysCpuPage;
4245	RTCCUINTREG uOldEFlags;
4246
4247	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4248	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4249	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4250	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4251
4252	#ifdef VBOX_STRICT
4253	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4254	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4255
4256	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4257	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4258	#endif
4259
4260	/* Disable interrupts. */
4261	uOldEFlags = ASMIntDisableFlags();
4262
4263	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4264	RTCPUID idHostCpu = RTMpCpuId();
4265	CPUMR0SetLApic(pVM, idHostCpu);
4266	#endif
4267
4268	pCpu = HMR0GetCurrentCpu();
4269	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4270
4271	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4272	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4273
4274	/* Leave VMX Root Mode. */
4275	VMXDisable();
4276
4277	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4278
4279	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4280	CPUMSetHyperEIP(pVCpu, enmOp);
4281	for (int i = (int)cbParam - 1; i >= 0; i--)
4282	CPUMPushHyper(pVCpu, paParam[i]);
4283
4284	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4285
4286	/* Call the switcher. */
4287	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4288	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4289
4290	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4291	/* Make sure the VMX instructions don't cause #UD faults. */
4292	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4293
4294	/* Re-enter VMX Root Mode */
4295	rc2 = VMXEnable(HCPhysCpuPage);
4296	if (RT_FAILURE(rc2))
4297	{
4298	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4299	ASMSetFlags(uOldEFlags);
4300	return rc2;
4301	}
4302
4303	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4304	AssertRC(rc2);
4305	Assert(!(ASMGetFlags() & X86_EFL_IF));
4306	ASMSetFlags(uOldEFlags);
4307	return rc;
4308	}
4309
4310
4311	/**
4312	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4313	* supporting 64-bit guests.
4314	*
4315	* @returns VBox status code.
4316	* @param fResume Whether to VMLAUNCH or VMRESUME.
4317	* @param pCtx Pointer to the guest-CPU context.
4318	* @param pCache Pointer to the VMCS cache.
4319	* @param pVM Pointer to the VM.
4320	* @param pVCpu Pointer to the VMCPU.
4321	*/
4322	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4323	{
4324	uint32_t aParam[6];
4325	PHMGLOBLCPUINFO pCpu = NULL;
4326	RTHCPHYS HCPhysCpuPage = 0;
4327	int rc = VERR_INTERNAL_ERROR_5;
4328
4329	pCpu = HMR0GetCurrentCpu();
4330	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4331
4332	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4333	pCache->uPos = 1;
4334	pCache->interPD = PGMGetInterPaeCR3(pVM);
4335	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4336	#endif
4337
4338	#ifdef VBOX_STRICT
4339	pCache->TestIn.HCPhysCpuPage = 0;
4340	pCache->TestIn.HCPhysVmcs = 0;
4341	pCache->TestIn.pCache = 0;
4342	pCache->TestOut.HCPhysVmcs = 0;
4343	pCache->TestOut.pCache = 0;
4344	pCache->TestOut.pCtx = 0;
4345	pCache->TestOut.eflags = 0;
4346	#endif
4347
4348	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4349	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4350	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4351	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4352	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4353	aParam[5] = 0;
4354
4355	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4356	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4357	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4358	#endif
4359	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4360
4361	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4362	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4363	Assert(pCtx->dr[4] == 10);
4364	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4365	#endif
4366
4367	#ifdef VBOX_STRICT
4368	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4369	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4370	pVCpu->hm.s.vmx.HCPhysVmcs));
4371	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4372	pCache->TestOut.HCPhysVmcs));
4373	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4374	pCache->TestOut.pCache));
4375	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4376	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4377	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4378	pCache->TestOut.pCtx));
4379	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4380	#endif
4381	return rc;
4382	}
4383
4384
4385	/**
4386	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4387	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4388	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4389	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4390	*
4391	* @returns VBox status code.
4392	* @param pVM Pointer to the VM.
4393	* @param pVCpu Pointer to the VMCPU.
4394	*/
4395	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4396	{
4397	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4398	{ \
4399	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4400	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4401	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4402	++cReadFields; \
4403	}
4404
4405	AssertPtr(pVM);
4406	AssertPtr(pVCpu);
4407	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4408	uint32_t cReadFields = 0;
4409
4410	/*
4411	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4412	* and serve to indicate exceptions to the rules.
4413	*/
4414
4415	/* Guest-natural selector base fields. */
4416	#if 0
4417	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4418	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4419	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4420	#endif
4421	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4422	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4423	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4424	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4425	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4426	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4427	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4428	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4429	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4430	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4431	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4432	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4433	#if 0
4434	/* Unused natural width guest-state fields. */
4435	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4436	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4437	#endif
4438	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4439	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4440
4441	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4442	#if 0
4443	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4444	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4445	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4446	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4447	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4448	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4449	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4450	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4451	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4452	#endif
4453
4454	/* Natural width guest-state fields. */
4455	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4456	#if 0
4457	/* Currently unused field. */
4458	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4459	#endif
4460
4461	if (pVM->hm.s.fNestedPaging)
4462	{
4463	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4464	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4465	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4466	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4467	}
4468	else
4469	{
4470	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4471	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4472	}
4473
4474	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4475	return VINF_SUCCESS;
4476	}
4477
4478
4479	/**
4480	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4481	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4482	* darwin, running 64-bit guests).
4483	*
4484	* @returns VBox status code.
4485	* @param pVCpu Pointer to the VMCPU.
4486	* @param idxField The VMCS field encoding.
4487	* @param u64Val 16, 32 or 64 bits value.
4488	*/
4489	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4490	{
4491	int rc;
4492	switch (idxField)
4493	{
4494	/*
4495	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4496	*/
4497	/* 64-bit Control fields. */
4498	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4499	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4500	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4501	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4502	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4503	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4504	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4505	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4506	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4507	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4508	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4509	case VMX_VMCS64_CTRL_EPTP_FULL:
4510	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4511	/* 64-bit Guest-state fields. */
4512	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4513	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4514	case VMX_VMCS64_GUEST_PAT_FULL:
4515	case VMX_VMCS64_GUEST_EFER_FULL:
4516	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4517	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4518	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4519	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4520	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4521	/* 64-bit Host-state fields. */
4522	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4523	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4524	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4525	{
4526	rc = VMXWriteVmcs32(idxField, u64Val);
4527	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4528	break;
4529	}
4530
4531	/*
4532	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4533	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4534	*/
4535	/* Natural-width Guest-state fields. */
4536	case VMX_VMCS_GUEST_CR3:
4537	case VMX_VMCS_GUEST_ES_BASE:
4538	case VMX_VMCS_GUEST_CS_BASE:
4539	case VMX_VMCS_GUEST_SS_BASE:
4540	case VMX_VMCS_GUEST_DS_BASE:
4541	case VMX_VMCS_GUEST_FS_BASE:
4542	case VMX_VMCS_GUEST_GS_BASE:
4543	case VMX_VMCS_GUEST_LDTR_BASE:
4544	case VMX_VMCS_GUEST_TR_BASE:
4545	case VMX_VMCS_GUEST_GDTR_BASE:
4546	case VMX_VMCS_GUEST_IDTR_BASE:
4547	case VMX_VMCS_GUEST_RSP:
4548	case VMX_VMCS_GUEST_RIP:
4549	case VMX_VMCS_GUEST_SYSENTER_ESP:
4550	case VMX_VMCS_GUEST_SYSENTER_EIP:
4551	{
4552	if (!(u64Val >> 32))
4553	{
4554	/* If this field is 64-bit, VT-x will zero out the top bits. */
4555	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4556	}
4557	else
4558	{
4559	/* Assert that only the 32->64 switcher case should ever come here. */
4560	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4561	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4562	}
4563	break;
4564	}
4565
4566	default:
4567	{
4568	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4569	rc = VERR_INVALID_PARAMETER;
4570	break;
4571	}
4572	}
4573	AssertRCReturn(rc, rc);
4574	return rc;
4575	}
4576
4577
4578	/**
4579	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4580	* hosts (except darwin) for 64-bit guests.
4581	*
4582	* @param pVCpu Pointer to the VMCPU.
4583	* @param idxField The VMCS field encoding.
4584	* @param u64Val 16, 32 or 64 bits value.
4585	*/
4586	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4587	{
4588	AssertPtr(pVCpu);
4589	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4590
4591	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4592	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4593
4594	/* Make sure there are no duplicates. */
4595	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4596	{
4597	if (pCache->Write.aField[i] == idxField)
4598	{
4599	pCache->Write.aFieldVal[i] = u64Val;
4600	return VINF_SUCCESS;
4601	}
4602	}
4603
4604	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4605	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4606	pCache->Write.cValidEntries++;
4607	return VINF_SUCCESS;
4608	}
4609
4610	/* Enable later when the assembly code uses these as callbacks. */
4611	#if 0
4612	/*
4613	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4614	*
4615	* @param pVCpu Pointer to the VMCPU.
4616	* @param pCache Pointer to the VMCS cache.
4617	*
4618	* @remarks No-long-jump zone!!!
4619	*/
4620	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4621	{
4622	AssertPtr(pCache);
4623	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4624	{
4625	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4626	AssertRC(rc);
4627	}
4628	pCache->Write.cValidEntries = 0;
4629	}
4630
4631
4632	/**
4633	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4634	*
4635	* @param pVCpu Pointer to the VMCPU.
4636	* @param pCache Pointer to the VMCS cache.
4637	*
4638	* @remarks No-long-jump zone!!!
4639	*/
4640	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4641	{
4642	AssertPtr(pCache);
4643	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4644	{
4645	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4646	AssertRC(rc);
4647	}
4648	}
4649	#endif
4650	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4651
4652
4653	/**
4654	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4655	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4656	* timer.
4657	*
4658	* @returns VBox status code.
4659	* @param pVCpu Pointer to the VMCPU.
4660	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4661	* out-of-sync. Make sure to update the required fields
4662	* before using them.
4663	* @remarks No-long-jump zone!!!
4664	*/
4665	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4666	{
4667	int rc = VERR_INTERNAL_ERROR_5;
4668	bool fOffsettedTsc = false;
4669	PVM pVM = pVCpu->CTX_SUFF(pVM);
4670	if (pVM->hm.s.vmx.fUsePreemptTimer)
4671	{
4672	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4673
4674	/* Make sure the returned values have sane upper and lower boundaries. */
4675	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4676	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4677	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4678	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4679
4680	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4681	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4682	}
4683	else
4684	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4685
4686	if (fOffsettedTsc)
4687	{
4688	uint64_t u64CurTSC = ASMReadTSC();
4689	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4690	{
4691	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4692	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4693
4694	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4695	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4696	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4697	}
4698	else
4699	{
4700	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4701	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4702	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4703	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4704	}
4705	}
4706	else
4707	{
4708	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4709	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4710	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4711	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4712	}
4713	}
4714
4715
4716	/**
4717	* Determines if an exception is a contributory exception. Contributory
4718	* exceptions are ones which can cause double-faults. Page-fault is
4719	* intentionally not included here as it's a conditional contributory exception.
4720	*
4721	* @returns true if the exception is contributory, false otherwise.
4722	* @param uVector The exception vector.
4723	*/
4724	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4725	{
4726	switch (uVector)
4727	{
4728	case X86_XCPT_GP:
4729	case X86_XCPT_SS:
4730	case X86_XCPT_NP:
4731	case X86_XCPT_TS:
4732	case X86_XCPT_DE:
4733	return true;
4734	default:
4735	break;
4736	}
4737	return false;
4738	}
4739
4740
4741	/**
4742	* Sets an event as a pending event to be injected into the guest.
4743	*
4744	* @param pVCpu Pointer to the VMCPU.
4745	* @param u32IntrInfo The VM-entry interruption-information field.
4746	* @param cbInstr The VM-entry instruction length in bytes (for software
4747	* interrupts, exceptions and privileged software
4748	* exceptions).
4749	* @param u32ErrCode The VM-entry exception error code.
4750	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4751	* page-fault.
4752	*
4753	* @remarks Statistics counter assumes this is a guest event being injected or
4754	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4755	* always incremented.
4756	*/
4757	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4758	RTGCUINTPTR GCPtrFaultAddress)
4759	{
4760	Assert(!pVCpu->hm.s.Event.fPending);
4761	pVCpu->hm.s.Event.fPending = true;
4762	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4763	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4764	pVCpu->hm.s.Event.cbInstr = cbInstr;
4765	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4766
4767	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4768	}
4769
4770
4771	/**
4772	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4773	*
4774	* @param pVCpu Pointer to the VMCPU.
4775	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4776	* out-of-sync. Make sure to update the required fields
4777	* before using them.
4778	*/
4779	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4780	{
4781	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4782	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4783	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4784	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4785	}
4786
4787
4788	/**
4789	* Handle a condition that occurred while delivering an event through the guest
4790	* IDT.
4791	*
4792	* @returns VBox status code (informational error codes included).
4793	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4794	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4795	* continue execution of the guest which will delivery the #DF.
4796	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4797	*
4798	* @param pVCpu Pointer to the VMCPU.
4799	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4800	* out-of-sync. Make sure to update the required fields
4801	* before using them.
4802	* @param pVmxTransient Pointer to the VMX transient structure.
4803	*
4804	* @remarks No-long-jump zone!!!
4805	*/
4806	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4807	{
4808	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4809	AssertRC(rc);
4810	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4811	{
4812	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4813	AssertRCReturn(rc, rc);
4814
4815	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4816	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4817	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4818
4819	typedef enum
4820	{
4821	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4822	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4823	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4824	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4825	} VMXREFLECTXCPT;
4826
4827	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4828	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4829	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4830	{
4831	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4832	{
4833	enmReflect = VMXREFLECTXCPT_XCPT;
4834	#ifdef VBOX_STRICT
4835	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4836	&& uExitVector == X86_XCPT_PF)
4837	{
4838	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4839	}
4840	#endif
4841	if ( uExitVector == X86_XCPT_PF
4842	&& uIdtVector == X86_XCPT_PF)
4843	{
4844	pVmxTransient->fVectoringPF = true;
4845	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4846	}
4847	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4848	&& hmR0VmxIsContributoryXcpt(uExitVector)
4849	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4850	\|\| uIdtVector == X86_XCPT_PF))
4851	{
4852	enmReflect = VMXREFLECTXCPT_DF;
4853	}
4854	else if (uIdtVector == X86_XCPT_DF)
4855	enmReflect = VMXREFLECTXCPT_TF;
4856	}
4857	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4858	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4859	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4860	{
4861	/*
4862	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4863	* (whatever they are) as they reoccur when restarting the instruction.
4864	*/
4865	enmReflect = VMXREFLECTXCPT_XCPT;
4866	}
4867	}
4868	else
4869	{
4870	/*
4871	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
4872	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
4873	* original exception to the guest after handling the VM-exit.
4874	*/
4875	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4876	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4877	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4878	{
4879	enmReflect = VMXREFLECTXCPT_XCPT;
4880	}
4881	}
4882
4883	switch (enmReflect)
4884	{
4885	case VMXREFLECTXCPT_XCPT:
4886	{
4887	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4888	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4889	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
4890
4891	uint32_t u32ErrCode = 0;
4892	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
4893	{
4894	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4895	AssertRCReturn(rc, rc);
4896	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4897	}
4898
4899	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4900	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4901	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4902	rc = VINF_SUCCESS;
4903	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
4904	pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4905
4906	break;
4907	}
4908
4909	case VMXREFLECTXCPT_DF:
4910	{
4911	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4912	rc = VINF_HM_DOUBLE_FAULT;
4913	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
4914	pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4915
4916	break;
4917	}
4918
4919	case VMXREFLECTXCPT_TF:
4920	{
4921	rc = VINF_EM_RESET;
4922	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
4923	uExitVector));
4924	break;
4925	}
4926
4927	default:
4928	Assert(rc == VINF_SUCCESS);
4929	break;
4930	}
4931	}
4932	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4933	return rc;
4934	}
4935
4936
4937	/**
4938	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4939	*
4940	* @returns VBox status code.
4941	* @param pVCpu Pointer to the VMCPU.
4942	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4943	* out-of-sync. Make sure to update the required fields
4944	* before using them.
4945	*
4946	* @remarks No-long-jump zone!!!
4947	*/
4948	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4949	{
4950	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4951	{
4952	uint32_t uVal = 0;
4953	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4954	AssertRCReturn(rc, rc);
4955	uint32_t uShadow = 0;
4956	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4957	AssertRCReturn(rc, rc);
4958
4959	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
4960	CPUMSetGuestCR0(pVCpu, uVal);
4961	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4962	}
4963	return VINF_SUCCESS;
4964	}
4965
4966
4967	/**
4968	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4969	*
4970	* @returns VBox status code.
4971	* @param pVCpu Pointer to the VMCPU.
4972	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4973	* out-of-sync. Make sure to update the required fields
4974	* before using them.
4975	*
4976	* @remarks No-long-jump zone!!!
4977	*/
4978	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4979	{
4980	int rc = VINF_SUCCESS;
4981	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4982	{
4983	uint32_t uVal = 0;
4984	uint32_t uShadow = 0;
4985	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4986	AssertRCReturn(rc, rc);
4987	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4988	AssertRCReturn(rc, rc);
4989
4990	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
4991	CPUMSetGuestCR4(pVCpu, uVal);
4992	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4993	}
4994	return rc;
4995	}
4996
4997
4998	/**
4999	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5000	*
5001	* @returns VBox status code.
5002	* @param pVCpu Pointer to the VMCPU.
5003	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5004	* out-of-sync. Make sure to update the required fields
5005	* before using them.
5006	*
5007	* @remarks No-long-jump zone!!!
5008	*/
5009	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5010	{
5011	int rc = VINF_SUCCESS;
5012	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5013	{
5014	uint64_t u64Val = 0;
5015	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5016	AssertRCReturn(rc, rc);
5017
5018	pMixedCtx->rip = u64Val;
5019	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5020	}
5021	return rc;
5022	}
5023
5024
5025	/**
5026	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5027	*
5028	* @returns VBox status code.
5029	* @param pVCpu Pointer to the VMCPU.
5030	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5031	* out-of-sync. Make sure to update the required fields
5032	* before using them.
5033	*
5034	* @remarks No-long-jump zone!!!
5035	*/
5036	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5037	{
5038	int rc = VINF_SUCCESS;
5039	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5040	{
5041	uint64_t u64Val = 0;
5042	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5043	AssertRCReturn(rc, rc);
5044
5045	pMixedCtx->rsp = u64Val;
5046	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5047	}
5048	return rc;
5049	}
5050
5051
5052	/**
5053	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5054	*
5055	* @returns VBox status code.
5056	* @param pVCpu Pointer to the VMCPU.
5057	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5058	* out-of-sync. Make sure to update the required fields
5059	* before using them.
5060	*
5061	* @remarks No-long-jump zone!!!
5062	*/
5063	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5064	{
5065	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5066	{
5067	uint32_t uVal = 0;
5068	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5069	AssertRCReturn(rc, rc);
5070
5071	pMixedCtx->eflags.u32 = uVal;
5072	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5073	{
5074	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5075	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5076
5077	pMixedCtx->eflags.Bits.u1VM = 0;
5078	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
5079	}
5080
5081	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5082	}
5083	return VINF_SUCCESS;
5084	}
5085
5086
5087	/**
5088	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5089	* guest-CPU context.
5090	*/
5091	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5092	{
5093	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5094	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5095	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5096	return rc;
5097	}
5098
5099
5100	/**
5101	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5102	* from the guest-state area in the VMCS.
5103	*
5104	* @param pVCpu Pointer to the VMCPU.
5105	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5106	* out-of-sync. Make sure to update the required fields
5107	* before using them.
5108	*
5109	* @remarks No-long-jump zone!!!
5110	*/
5111	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5112	{
5113	uint32_t uIntrState = 0;
5114	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5115	AssertRC(rc);
5116
5117	if (!uIntrState)
5118	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5119	else
5120	{
5121	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5122	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5123	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5124	AssertRC(rc);
5125	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5126	AssertRC(rc);
5127
5128	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5129	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5130	}
5131	}
5132
5133
5134	/**
5135	* Saves the guest's activity state.
5136	*
5137	* @returns VBox status code.
5138	* @param pVCpu Pointer to the VMCPU.
5139	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5140	* out-of-sync. Make sure to update the required fields
5141	* before using them.
5142	*
5143	* @remarks No-long-jump zone!!!
5144	*/
5145	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5146	{
5147	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5148	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5149	return VINF_SUCCESS;
5150	}
5151
5152
5153	/**
5154	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5155	* the current VMCS into the guest-CPU context.
5156	*
5157	* @returns VBox status code.
5158	* @param pVCpu Pointer to the VMCPU.
5159	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5160	* out-of-sync. Make sure to update the required fields
5161	* before using them.
5162	*
5163	* @remarks No-long-jump zone!!!
5164	*/
5165	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5166	{
5167	int rc = VINF_SUCCESS;
5168	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5169	{
5170	uint32_t u32Val = 0;
5171	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5172	pMixedCtx->SysEnter.cs = u32Val;
5173	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5174	}
5175
5176	uint64_t u64Val = 0;
5177	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5178	{
5179	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5180	pMixedCtx->SysEnter.eip = u64Val;
5181	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5182	}
5183	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5184	{
5185	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5186	pMixedCtx->SysEnter.esp = u64Val;
5187	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5188	}
5189	return rc;
5190	}
5191
5192
5193	/**
5194	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5195	* context.
5196	*
5197	* @returns VBox status code.
5198	* @param pVCpu Pointer to the VMCPU.
5199	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5200	* out-of-sync. Make sure to update the required fields
5201	* before using them.
5202	*
5203	* @remarks No-long-jump zone!!!
5204	*/
5205	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5206	{
5207	int rc = VINF_SUCCESS;
5208	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5209	{
5210	uint64_t u64Val = 0;
5211	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5212	pMixedCtx->fs.u64Base = u64Val;
5213	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5214	}
5215	return rc;
5216	}
5217
5218
5219	/**
5220	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5221	* context.
5222	*
5223	* @returns VBox status code.
5224	* @param pVCpu Pointer to the VMCPU.
5225	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5226	* out-of-sync. Make sure to update the required fields
5227	* before using them.
5228	*
5229	* @remarks No-long-jump zone!!!
5230	*/
5231	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5232	{
5233	int rc = VINF_SUCCESS;
5234	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5235	{
5236	uint64_t u64Val = 0;
5237	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5238	pMixedCtx->gs.u64Base = u64Val;
5239	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5240	}
5241	return rc;
5242	}
5243
5244
5245	/**
5246	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5247	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5248	* and TSC_AUX.
5249	*
5250	* @returns VBox status code.
5251	* @param pVCpu Pointer to the VMCPU.
5252	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5253	* out-of-sync. Make sure to update the required fields
5254	* before using them.
5255	*
5256	* @remarks No-long-jump zone!!!
5257	*/
5258	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5259	{
5260	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5261	return VINF_SUCCESS;
5262
5263	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5264	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5265	{
5266	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5267	pMsr += i;
5268	switch (pMsr->u32IndexMSR)
5269	{
5270	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5271	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5272	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5273	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5274	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5275	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5276	default:
5277	{
5278	AssertFailed();
5279	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5280	}
5281	}
5282	}
5283	#endif
5284
5285	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5286	return VINF_SUCCESS;
5287	}
5288
5289
5290	/**
5291	* Saves the guest control registers from the current VMCS into the guest-CPU
5292	* context.
5293	*
5294	* @returns VBox status code.
5295	* @param pVCpu Pointer to the VMCPU.
5296	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5297	* out-of-sync. Make sure to update the required fields
5298	* before using them.
5299	*
5300	* @remarks No-long-jump zone!!!
5301	*/
5302	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5303	{
5304	/* Guest CR0. Guest FPU. */
5305	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5306	AssertRCReturn(rc, rc);
5307
5308	/* Guest CR4. */
5309	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5310	AssertRCReturn(rc, rc);
5311
5312	/* Guest CR2 - updated always during the world-switch or in #PF. */
5313	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5314	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5315	{
5316	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5317	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5318
5319	PVM pVM = pVCpu->CTX_SUFF(pVM);
5320	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5321	\|\| ( pVM->hm.s.fNestedPaging
5322	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5323	{
5324	uint64_t u64Val = 0;
5325	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5326	if (pMixedCtx->cr3 != u64Val)
5327	{
5328	CPUMSetGuestCR3(pVCpu, u64Val);
5329	if (VMMRZCallRing3IsEnabled(pVCpu))
5330	{
5331	PGMUpdateCR3(pVCpu, u64Val);
5332	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5333	}
5334	else
5335	{
5336	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5337	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5338	}
5339	}
5340
5341	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5342	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5343	{
5344	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5345	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5346	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5347	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5348
5349	if (VMMRZCallRing3IsEnabled(pVCpu))
5350	{
5351	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5352	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5353	}
5354	else
5355	{
5356	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5357	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5358	}
5359	}
5360	}
5361
5362	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5363	}
5364
5365	/*
5366	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5367	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5368	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5369	*
5370	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5371	*/
5372	if (VMMRZCallRing3IsEnabled(pVCpu))
5373	{
5374	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5375	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5376
5377	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5378	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5379
5380	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5381	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5382	}
5383
5384	return rc;
5385	}
5386
5387
5388	/**
5389	* Reads a guest segment register from the current VMCS into the guest-CPU
5390	* context.
5391	*
5392	* @returns VBox status code.
5393	* @param pVCpu Pointer to the VMCPU.
5394	* @param idxSel Index of the selector in the VMCS.
5395	* @param idxLimit Index of the segment limit in the VMCS.
5396	* @param idxBase Index of the segment base in the VMCS.
5397	* @param idxAccess Index of the access rights of the segment in the VMCS.
5398	* @param pSelReg Pointer to the segment selector.
5399	*
5400	* @remarks No-long-jump zone!!!
5401	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5402	* macro as that takes care of whether to read from the VMCS cache or
5403	* not.
5404	*/
5405	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5406	PCPUMSELREG pSelReg)
5407	{
5408	uint32_t u32Val = 0;
5409	int rc = VMXReadVmcs32(idxSel, &u32Val);
5410	AssertRCReturn(rc, rc);
5411	pSelReg->Sel = (uint16_t)u32Val;
5412	pSelReg->ValidSel = (uint16_t)u32Val;
5413	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5414
5415	rc = VMXReadVmcs32(idxLimit, &u32Val);
5416	AssertRCReturn(rc, rc);
5417	pSelReg->u32Limit = u32Val;
5418
5419	uint64_t u64Val = 0;
5420	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5421	AssertRCReturn(rc, rc);
5422	pSelReg->u64Base = u64Val;
5423
5424	rc = VMXReadVmcs32(idxAccess, &u32Val);
5425	AssertRCReturn(rc, rc);
5426	pSelReg->Attr.u = u32Val;
5427
5428	/*
5429	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5430	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5431	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5432	*
5433	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5434	*
5435	* bird: This isn't quite as simple. VT-x and VBox(!) requires the DPL for SS to be the the same as CPL. In 64-bit mode it
5436	* is possible (int/trap/xxx injects does this when switching rings) to load SS with a NULL selector and RPL=CPL.
5437	* The Attr.u = X86DESCATTR_UNUSABLE works fine as long as nobody uses ring-1 or ring-2. VT-x seems to set the DPL
5438	* correctly in the attributes even when the unusable bit is set, we need to preseve the DPL or we get invalid guest
5439	* state trouble. Try bs2-cpu-hidden-regs-1.
5440	*/
5441	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5442	{
5443	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5444	Log(("idxSel=%#x attr=%#x\n", idxSel, pSelReg->Attr.u));
5445
5446	if (idxSel == VMX_VMCS16_GUEST_FIELD_SS)
5447	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_DPL;
5448	else if (idxSel == VMX_VMCS16_GUEST_FIELD_CS)
5449	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G;
5450	else
5451	pSelReg->Attr.u = X86DESCATTR_UNUSABLE;
5452	}
5453	return VINF_SUCCESS;
5454	}
5455
5456
5457	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5458	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5459	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5460	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5461	#else
5462	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5463	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5464	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5465	#endif
5466
5467
5468	/**
5469	* Saves the guest segment registers from the current VMCS into the guest-CPU
5470	* context.
5471	*
5472	* @returns VBox status code.
5473	* @param pVCpu Pointer to the VMCPU.
5474	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5475	* out-of-sync. Make sure to update the required fields
5476	* before using them.
5477	*
5478	* @remarks No-long-jump zone!!!
5479	*/
5480	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5481	{
5482	/* Guest segment registers. */
5483	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5484	{
5485	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5486	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5487	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5488	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5489	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5490	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5491	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5492
5493	/* Restore segment attributes for real-on-v86 mode hack. */
5494	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5495	{
5496	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5497	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5498	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5499	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5500	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5501	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5502	}
5503	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5504	}
5505
5506	return VINF_SUCCESS;
5507	}
5508
5509
5510	/**
5511	* Saves the guest descriptor table registers and task register from the current
5512	* VMCS into the guest-CPU context.
5513	*
5514	* @returns VBox status code.
5515	* @param pVCpu Pointer to the VMCPU.
5516	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5517	* out-of-sync. Make sure to update the required fields
5518	* before using them.
5519	*
5520	* @remarks No-long-jump zone!!!
5521	*/
5522	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5523	{
5524	int rc = VINF_SUCCESS;
5525
5526	/* Guest LDTR. */
5527	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5528	{
5529	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5530	AssertRCReturn(rc, rc);
5531	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5532	}
5533
5534	/* Guest GDTR. */
5535	uint64_t u64Val = 0;
5536	uint32_t u32Val = 0;
5537	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5538	{
5539	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5540	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5541	pMixedCtx->gdtr.pGdt = u64Val;
5542	pMixedCtx->gdtr.cbGdt = u32Val;
5543	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5544	}
5545
5546	/* Guest IDTR. */
5547	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5548	{
5549	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5550	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5551	pMixedCtx->idtr.pIdt = u64Val;
5552	pMixedCtx->idtr.cbIdt = u32Val;
5553	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5554	}
5555
5556	/* Guest TR. */
5557	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5558	{
5559	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5560	AssertRCReturn(rc, rc);
5561
5562	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5563	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5564	{
5565	rc = VMXLOCAL_READ_SEG(TR, tr);
5566	AssertRCReturn(rc, rc);
5567	}
5568	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5569	}
5570	return rc;
5571	}
5572
5573	#undef VMXLOCAL_READ_SEG
5574
5575
5576	/**
5577	* Saves the guest debug registers from the current VMCS into the guest-CPU
5578	* context.
5579	*
5580	* @returns VBox status code.
5581	* @param pVCpu Pointer to the VMCPU.
5582	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5583	* out-of-sync. Make sure to update the required fields
5584	* before using them.
5585	*
5586	* @remarks No-long-jump zone!!!
5587	*/
5588	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5589	{
5590	int rc = VINF_SUCCESS;
5591	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5592	{
5593	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5594	uint32_t u32Val;
5595	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5596	pMixedCtx->dr[7] = u32Val;
5597
5598	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5599	}
5600	return rc;
5601	}
5602
5603
5604	/**
5605	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
5606	*
5607	* @returns VBox status code.
5608	* @param pVCpu Pointer to the VMCPU.
5609	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5610	* out-of-sync. Make sure to update the required fields
5611	* before using them.
5612	*
5613	* @remarks No-long-jump zone!!!
5614	*/
5615	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5616	{
5617	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5618	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5619	return VINF_SUCCESS;
5620	}
5621
5622
5623	/**
5624	* Saves the entire guest state from the currently active VMCS into the
5625	* guest-CPU context. This essentially VMREADs all guest-data.
5626	*
5627	* @returns VBox status code.
5628	* @param pVCpu Pointer to the VMCPU.
5629	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5630	* out-of-sync. Make sure to update the required fields
5631	* before using them.
5632	*/
5633	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5634	{
5635	Assert(pVCpu);
5636	Assert(pMixedCtx);
5637
5638	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5639	return VINF_SUCCESS;
5640
5641	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5642	there is no real need to. */
5643	if (VMMRZCallRing3IsEnabled(pVCpu))
5644	VMMR0LogFlushDisable(pVCpu);
5645	else
5646	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5647	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5648
5649	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5650	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5651
5652	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5653	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5654
5655	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5656	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5657
5658	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5659	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5660
5661	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5662	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5663
5664	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5665	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5666
5667	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5668	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5669
5670	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5671	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5672
5673	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5674	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5675
5676	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5677	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5678
5679	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5680	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5681
5682	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5683	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5684
5685	if (VMMRZCallRing3IsEnabled(pVCpu))
5686	VMMR0LogFlushEnable(pVCpu);
5687
5688	return rc;
5689	}
5690
5691
5692	/**
5693	* Check per-VM and per-VCPU force flag actions that require us to go back to
5694	* ring-3 for one reason or another.
5695	*
5696	* @returns VBox status code (information status code included).
5697	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5698	* ring-3.
5699	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5700	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5701	* interrupts)
5702	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5703	* all EMTs to be in ring-3.
5704	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5705	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5706	* to the EM loop.
5707	*
5708	* @param pVM Pointer to the VM.
5709	* @param pVCpu Pointer to the VMCPU.
5710	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5711	* out-of-sync. Make sure to update the required fields
5712	* before using them.
5713	*/
5714	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5715	{
5716	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5717
5718	int rc = VERR_INTERNAL_ERROR_5;
5719	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5720	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5721	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5722	{
5723	/* We need the control registers now, make sure the guest-CPU context is updated. */
5724	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5725	AssertRCReturn(rc, rc);
5726
5727	/* Pending HM CR3 sync. */
5728	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5729	{
5730	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5731	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5732	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5733	}
5734
5735	/* Pending HM PAE PDPEs. */
5736	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5737	{
5738	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5739	AssertRC(rc);
5740	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5741	}
5742
5743	/* Pending PGM C3 sync. */
5744	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5745	{
5746	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5747	if (rc != VINF_SUCCESS)
5748	{
5749	AssertRC(rc);
5750	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5751	return rc;
5752	}
5753	}
5754
5755	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5756	/* -XXX- what was that about single stepping? */
5757	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5758	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5759	{
5760	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5761	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5762	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5763	return rc;
5764	}
5765
5766	/* Pending VM request packets, such as hardware interrupts. */
5767	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5768	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5769	{
5770	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5771	return VINF_EM_PENDING_REQUEST;
5772	}
5773
5774	/* Pending PGM pool flushes. */
5775	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5776	{
5777	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5778	return VINF_PGM_POOL_FLUSH_PENDING;
5779	}
5780
5781	/* Pending DMA requests. */
5782	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5783	{
5784	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5785	return VINF_EM_RAW_TO_R3;
5786	}
5787	}
5788
5789	/* Paranoia. */
5790	Assert(rc != VERR_EM_INTERPRETER);
5791	return VINF_SUCCESS;
5792	}
5793
5794
5795	/**
5796	* Converts any TRPM trap into a pending HM event. This is typically used when
5797	* entering from ring-3 (not longjmp returns).
5798	*
5799	* @param pVCpu Pointer to the VMCPU.
5800	*/
5801	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5802	{
5803	Assert(TRPMHasTrap(pVCpu));
5804	Assert(!pVCpu->hm.s.Event.fPending);
5805
5806	uint8_t uVector;
5807	TRPMEVENT enmTrpmEvent;
5808	RTGCUINT uErrCode;
5809	RTGCUINTPTR GCPtrFaultAddress;
5810	uint8_t cbInstr;
5811
5812	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5813	AssertRC(rc);
5814
5815	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5816	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5817	if (enmTrpmEvent == TRPM_TRAP)
5818	{
5819	switch (uVector)
5820	{
5821	case X86_XCPT_BP:
5822	case X86_XCPT_OF:
5823	{
5824	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5825	break;
5826	}
5827
5828	case X86_XCPT_PF:
5829	case X86_XCPT_DF:
5830	case X86_XCPT_TS:
5831	case X86_XCPT_NP:
5832	case X86_XCPT_SS:
5833	case X86_XCPT_GP:
5834	case X86_XCPT_AC:
5835	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5836	/* no break! */
5837	default:
5838	{
5839	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5840	break;
5841	}
5842	}
5843	}
5844	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5845	{
5846	if (uVector == X86_XCPT_NMI)
5847	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5848	else
5849	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5850	}
5851	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5852	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5853	else
5854	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5855
5856	rc = TRPMResetTrap(pVCpu);
5857	AssertRC(rc);
5858	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5859	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5860
5861	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5862	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
5863	}
5864
5865
5866	/**
5867	* Converts any pending HM event into a TRPM trap. Typically used when leaving
5868	* VT-x to execute any instruction.
5869	*
5870	* @param pvCpu Pointer to the VMCPU.
5871	*/
5872	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
5873	{
5874	Assert(pVCpu->hm.s.Event.fPending);
5875
5876	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5877	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5878	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5879	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5880
5881	/* If a trap was already pending, we did something wrong! */
5882	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5883
5884	TRPMEVENT enmTrapType;
5885	switch (uVectorType)
5886	{
5887	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5888	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5889	enmTrapType = TRPM_HARDWARE_INT;
5890	break;
5891	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5892	enmTrapType = TRPM_SOFTWARE_INT;
5893	break;
5894	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5895	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5896	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5897	enmTrapType = TRPM_TRAP;
5898	break;
5899	default:
5900	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5901	enmTrapType = TRPM_32BIT_HACK;
5902	break;
5903	}
5904
5905	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5906
5907	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5908	AssertRC(rc);
5909
5910	if (fErrorCodeValid)
5911	TRPMSetErrorCode(pVCpu, uErrorCode);
5912
5913	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5914	&& uVector == X86_XCPT_PF)
5915	{
5916	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5917	}
5918	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5919	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5920	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5921	{
5922	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5923	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5924	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5925	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5926	}
5927	pVCpu->hm.s.Event.fPending = false;
5928	}
5929
5930
5931	/**
5932	* Does the necessary state syncing before doing a longjmp to ring-3.
5933	*
5934	* @param pVM Pointer to the VM.
5935	* @param pVCpu Pointer to the VMCPU.
5936	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5937	* out-of-sync. Make sure to update the required fields
5938	* before using them.
5939	* @param rcExit The reason for exiting to ring-3. Can be
5940	* VINF_VMM_UNKNOWN_RING3_CALL.
5941	*
5942	* @remarks No-long-jmp zone!!!
5943	*/
5944	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5945	{
5946	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5947	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5948
5949	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5950	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5951	AssertRC(rc);
5952
5953	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
5954	if (CPUMIsGuestFPUStateActive(pVCpu))
5955	{
5956	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5957	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5958	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5959	}
5960
5961	/* Restore host debug registers if necessary and resync on next R0 reentry. */
5962	if (CPUMIsGuestDebugStateActive(pVCpu))
5963	{
5964	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5965	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5966	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5967	}
5968	else if (CPUMIsHyperDebugStateActive(pVCpu))
5969	{
5970	CPUMR0LoadHostDebugState(pVM, pVCpu);
5971	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5972	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5973	}
5974
5975	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
5976	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
5977	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
5978	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
5979	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
5980	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
5981	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
5982	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5983
5984	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5985	}
5986
5987
5988	/**
5989	* An action requires us to go back to ring-3. This function does the necessary
5990	* steps before we can safely return to ring-3. This is not the same as longjmps
5991	* to ring-3, this is voluntary.
5992	*
5993	* @param pVM Pointer to the VM.
5994	* @param pVCpu Pointer to the VMCPU.
5995	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5996	* out-of-sync. Make sure to update the required fields
5997	* before using them.
5998	* @param rcExit The reason for exiting to ring-3. Can be
5999	* VINF_VMM_UNKNOWN_RING3_CALL.
6000	*/
6001	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6002	{
6003	Assert(pVM);
6004	Assert(pVCpu);
6005	Assert(pMixedCtx);
6006	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6007
6008	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6009	{
6010	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
6011	return;
6012	}
6013	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6014	{
6015	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6016	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6017	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6018	pVCpu->hm.s.vmx.LastError.idCurrentCpu = RTMpCpuId();
6019	return;
6020	}
6021
6022	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6023	VMMRZCallRing3Disable(pVCpu);
6024	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6025
6026	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6027	if (pVCpu->hm.s.Event.fPending)
6028	{
6029	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6030	Assert(!pVCpu->hm.s.Event.fPending);
6031	}
6032
6033	/* Sync. the guest state. */
6034	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
6035	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6036
6037	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6038	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6039	\| CPUM_CHANGED_LDTR
6040	\| CPUM_CHANGED_GDTR
6041	\| CPUM_CHANGED_IDTR
6042	\| CPUM_CHANGED_TR
6043	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6044
6045	/* On our way back from ring-3 the following needs to be done. */
6046	/** @todo This can change with preemption hooks. */
6047	if (rcExit == VINF_EM_RAW_INTERRUPT)
6048	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
6049	else
6050	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
6051
6052	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6053	VMMRZCallRing3Enable(pVCpu);
6054	}
6055
6056
6057	/**
6058	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6059	* longjump to ring-3 and possibly get preempted.
6060	*
6061	* @param pVCpu Pointer to the VMCPU.
6062	* @param enmOperation The operation causing the ring-3 longjump.
6063	* @param pvUser The user argument (pointer to the possibly
6064	* out-of-date guest-CPU context).
6065	*
6066	* @remarks Must never be called with @a enmOperation ==
6067	* VMMCALLRING3_VM_R0_ASSERTION.
6068	*/
6069	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6070	{
6071	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6072	Assert(pVCpu);
6073	Assert(pvUser);
6074	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6075	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6076
6077	VMMRZCallRing3Disable(pVCpu);
6078	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6079	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6080	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
6081	VMMRZCallRing3Enable(pVCpu);
6082	}
6083
6084
6085	/**
6086	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6087	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6088	*
6089	* @param pVCpu Pointer to the VMCPU.
6090	*/
6091	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6092	{
6093	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6094	{
6095	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6096	{
6097	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6098	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6099	AssertRC(rc);
6100	}
6101	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6102	}
6103
6104
6105	/**
6106	* Injects any pending events into the guest if the guest is in a state to
6107	* receive them.
6108	*
6109	* @returns VBox status code (informational status codes included).
6110	* @param pVCpu Pointer to the VMCPU.
6111	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6112	* out-of-sync. Make sure to update the required fields
6113	* before using them.
6114	*/
6115	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6116	{
6117	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6118	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6119	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6120	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6121
6122	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6123	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6124	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6125	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6126	Assert(!TRPMHasTrap(pVCpu));
6127
6128	int rc = VINF_SUCCESS;
6129	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6130	{
6131	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6132	bool fInject = true;
6133	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6134	{
6135	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6136	AssertRCReturn(rc, rc);
6137	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6138	if ( fBlockInt
6139	\|\| fBlockSti
6140	\|\| fBlockMovSS)
6141	{
6142	fInject = false;
6143	}
6144	}
6145	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6146	&& ( fBlockMovSS
6147	\|\| fBlockSti))
6148	{
6149	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6150	fInject = false;
6151	}
6152
6153	if (fInject)
6154	{
6155	Log4(("Injecting pending event vcpu[%RU32]\n", pVCpu->idCpu));
6156	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6157	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6158	AssertRCReturn(rc, rc);
6159	pVCpu->hm.s.Event.fPending = false;
6160
6161	#ifdef VBOX_WITH_STATISTICS
6162	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6163	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6164	else
6165	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6166	#endif
6167	}
6168	else
6169	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6170	} /** @todo SMI. SMIs take priority over NMIs. */
6171	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6172	{
6173	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6174	if ( !fBlockMovSS
6175	&& !fBlockSti)
6176	{
6177	Log4(("Injecting NMI\n"));
6178	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6179	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6180	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6181	0 /* GCPtrFaultAddress */, &uIntrState);
6182	AssertRCReturn(rc, rc);
6183	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6184
6185	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6186	}
6187	else
6188	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6189	}
6190	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
6191	{
6192	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6193	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6194	AssertRCReturn(rc, rc);
6195	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6196	if ( !fBlockInt
6197	&& !fBlockSti
6198	&& !fBlockMovSS)
6199	{
6200	uint8_t u8Interrupt;
6201	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6202	if (RT_SUCCESS(rc))
6203	{
6204	Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6205	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6206	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6207	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6208	0 /* GCPtrFaultAddress */, &uIntrState);
6209
6210	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6211	}
6212	else
6213	{
6214	/** @todo Does this actually happen? If not turn it into an assertion. */
6215	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6216	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6217	rc = VINF_SUCCESS;
6218	}
6219	}
6220	else
6221	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6222	}
6223
6224	/*
6225	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6226	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6227	*/
6228	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6229	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6230	int rc2 = VINF_SUCCESS;
6231	if ( fBlockSti
6232	\|\| fBlockMovSS)
6233	{
6234	if (!DBGFIsStepping(pVCpu))
6235	{
6236	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6237	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6238	{
6239	/*
6240	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6241	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6242	*/
6243	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6244	AssertRCReturn(rc, rc);
6245	}
6246	}
6247	else
6248	{
6249	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6250	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6251	uIntrState = 0;
6252	}
6253	}
6254
6255	/*
6256	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6257	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6258	*/
6259	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6260	AssertRC(rc2);
6261
6262	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6263	return rc;
6264	}
6265
6266
6267	/**
6268	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6269	*
6270	* @param pVCpu Pointer to the VMCPU.
6271	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6272	* out-of-sync. Make sure to update the required fields
6273	* before using them.
6274	*/
6275	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6276	{
6277	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6278	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6279	}
6280
6281
6282	/**
6283	* Injects a double-fault (#DF) exception into the VM.
6284	*
6285	* @returns VBox status code (informational status code included).
6286	* @param pVCpu Pointer to the VMCPU.
6287	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6288	* out-of-sync. Make sure to update the required fields
6289	* before using them.
6290	*/
6291	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6292	{
6293	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6294	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6295	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6296	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6297	puIntrState);
6298	}
6299
6300
6301	/**
6302	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6303	*
6304	* @param pVCpu Pointer to the VMCPU.
6305	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6306	* out-of-sync. Make sure to update the required fields
6307	* before using them.
6308	*/
6309	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6310	{
6311	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6312	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6313	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6314	}
6315
6316
6317	/**
6318	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6319	*
6320	* @param pVCpu Pointer to the VMCPU.
6321	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6322	* out-of-sync. Make sure to update the required fields
6323	* before using them.
6324	* @param cbInstr The value of RIP that is to be pushed on the guest
6325	* stack.
6326	*/
6327	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6328	{
6329	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6330	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6331	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6332	}
6333
6334
6335	/**
6336	* Injects a general-protection (#GP) fault into the VM.
6337	*
6338	* @returns VBox status code (informational status code included).
6339	* @param pVCpu Pointer to the VMCPU.
6340	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6341	* out-of-sync. Make sure to update the required fields
6342	* before using them.
6343	* @param u32ErrorCode The error code associated with the #GP.
6344	*/
6345	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6346	uint32_t *puIntrState)
6347	{
6348	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6349	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6350	if (fErrorCodeValid)
6351	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6352	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6353	puIntrState);
6354	}
6355
6356
6357	/**
6358	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6359	*
6360	* @param pVCpu Pointer to the VMCPU.
6361	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6362	* out-of-sync. Make sure to update the required fields
6363	* before using them.
6364	* @param uVector The software interrupt vector number.
6365	* @param cbInstr The value of RIP that is to be pushed on the guest
6366	* stack.
6367	*/
6368	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6369	{
6370	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6371	if ( uVector == X86_XCPT_BP
6372	\|\| uVector == X86_XCPT_OF)
6373	{
6374	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6375	}
6376	else
6377	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6378	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6379	}
6380
6381
6382	/**
6383	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6384	* stack.
6385	*
6386	* @returns VBox status code (information status code included).
6387	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6388	* @param pVM Pointer to the VM.
6389	* @param pMixedCtx Pointer to the guest-CPU context.
6390	* @param uValue The value to push to the guest stack.
6391	*/
6392	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6393	{
6394	/*
6395	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6396	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6397	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6398	*/
6399	if (pMixedCtx->sp == 1)
6400	return VINF_EM_RESET;
6401	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6402	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6403	AssertRCReturn(rc, rc);
6404	return rc;
6405	}
6406
6407
6408	/**
6409	* Injects an event into the guest upon VM-entry by updating the relevant fields
6410	* in the VM-entry area in the VMCS.
6411	*
6412	* @returns VBox status code (informational error codes included).
6413	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6414	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6415	*
6416	* @param pVCpu Pointer to the VMCPU.
6417	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6418	* be out-of-sync. Make sure to update the required
6419	* fields before using them.
6420	* @param u64IntrInfo The VM-entry interruption-information field.
6421	* @param cbInstr The VM-entry instruction length in bytes (for
6422	* software interrupts, exceptions and privileged
6423	* software exceptions).
6424	* @param u32ErrCode The VM-entry exception error code.
6425	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6426	* @param puIntrState Pointer to the current guest interruptibility-state.
6427	* This interruptibility-state will be updated if
6428	* necessary. This cannot not be NULL.
6429	*
6430	* @remarks No-long-jump zone!!!
6431	* @remarks Requires CR0!
6432	*/
6433	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6434	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6435	{
6436	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6437	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6438	Assert(puIntrState);
6439	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6440
6441	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6442	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6443
6444	#ifdef VBOX_STRICT
6445	/* Validate the error-code-valid bit for hardware exceptions. */
6446	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6447	{
6448	switch (uVector)
6449	{
6450	case X86_XCPT_PF:
6451	case X86_XCPT_DF:
6452	case X86_XCPT_TS:
6453	case X86_XCPT_NP:
6454	case X86_XCPT_SS:
6455	case X86_XCPT_GP:
6456	case X86_XCPT_AC:
6457	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6458	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6459	/* fallthru */
6460	default:
6461	break;
6462	}
6463	}
6464	#endif
6465
6466	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6467	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6468	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6469
6470	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6471
6472	/* We require CR0 to check if the guest is in real-mode. */
6473	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6474	AssertRCReturn(rc, rc);
6475
6476	/*
6477	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6478	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6479	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6480	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6481	*/
6482	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6483	{
6484	PVM pVM = pVCpu->CTX_SUFF(pVM);
6485	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6486	{
6487	Assert(PDMVmmDevHeapIsEnabled(pVM));
6488	Assert(pVM->hm.s.vmx.pRealModeTSS);
6489
6490	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6491	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6492	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6493	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6494	AssertRCReturn(rc, rc);
6495	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6496
6497	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6498	const size_t cbIdtEntry = 4;
6499	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6500	{
6501	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6502	if (uVector == X86_XCPT_DF)
6503	return VINF_EM_RESET;
6504	else if (uVector == X86_XCPT_GP)
6505	{
6506	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6507	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6508	}
6509
6510	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6511	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6512	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6513	}
6514
6515	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6516	uint16_t uGuestIp = pMixedCtx->ip;
6517	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6518	{
6519	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6520	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6521	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6522	}
6523	else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6524	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6525
6526	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6527	uint16_t offIdtEntry = 0;
6528	RTSEL selIdtEntry = 0;
6529	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6530	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6531	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6532	AssertRCReturn(rc, rc);
6533
6534	/* Construct the stack frame for the interrupt/exception handler. */
6535	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6536	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6537	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6538	AssertRCReturn(rc, rc);
6539
6540	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6541	if (rc == VINF_SUCCESS)
6542	{
6543	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6544	pMixedCtx->rip = offIdtEntry;
6545	pMixedCtx->cs.Sel = selIdtEntry;
6546	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6547	if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6548	&& uVector == X86_XCPT_PF)
6549	{
6550	pMixedCtx->cr2 = GCPtrFaultAddress;
6551	}
6552	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6553	\| HM_CHANGED_GUEST_RIP
6554	\| HM_CHANGED_GUEST_RFLAGS
6555	\| HM_CHANGED_GUEST_RSP;
6556
6557	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6558	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6559	{
6560	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6561	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6562	Log4(("Clearing inhibition due to STI.\n"));
6563	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6564	}
6565	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6566	}
6567	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6568	return rc;
6569	}
6570	else
6571	{
6572	/*
6573	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6574	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6575	*/
6576	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6577	}
6578	}
6579
6580	/* Validate. */
6581	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6582	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6583	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6584
6585	/* Inject. */
6586	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6587	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6588	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6589	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6590
6591	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6592	&& uVector == X86_XCPT_PF)
6593	{
6594	pMixedCtx->cr2 = GCPtrFaultAddress;
6595	}
6596
6597	Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6598	u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6599
6600	AssertRCReturn(rc, rc);
6601	return rc;
6602	}
6603
6604
6605	/**
6606	* Enters the VT-x session.
6607	*
6608	* @returns VBox status code.
6609	* @param pVM Pointer to the VM.
6610	* @param pVCpu Pointer to the VMCPU.
6611	* @param pCpu Pointer to the CPU info struct.
6612	*/
6613	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6614	{
6615	AssertPtr(pVM);
6616	AssertPtr(pVCpu);
6617	Assert(pVM->hm.s.vmx.fSupported);
6618	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6619	NOREF(pCpu);
6620
6621	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6622
6623	/* Make sure we're in VMX root mode. */
6624	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6625	if (!(u32HostCR4 & X86_CR4_VMXE))
6626	{
6627	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6628	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6629	}
6630
6631	/* Load the active VMCS as the current one. */
6632	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6633	if (RT_FAILURE(rc))
6634	return rc;
6635
6636	/** @todo this will change with preemption hooks where can can VMRESUME as long
6637	* as we're no preempted. */
6638	pVCpu->hm.s.fResumeVM = false;
6639	return VINF_SUCCESS;
6640	}
6641
6642
6643	/**
6644	* Leaves the VT-x session.
6645	*
6646	* @returns VBox status code.
6647	* @param pVM Pointer to the VM.
6648	* @param pVCpu Pointer to the VMCPU.
6649	* @param pCtx Pointer to the guest-CPU context.
6650	*/
6651	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6652	{
6653	AssertPtr(pVCpu);
6654	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6655	NOREF(pVM);
6656	NOREF(pCtx);
6657
6658	/** @todo this will change with preemption hooks where we only VMCLEAR when
6659	* we are actually going to be preempted, not all the time like we
6660	* currently do. */
6661
6662	/* Restore host-state bits that VT-x only restores partially. */
6663	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6664	{
6665	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6666	/** @todo r=ramshankar: This is broken when
6667	* VBOX_WITH_VMMR0_DISABLE_PREEMPTION is not defined. As
6668	* VMXRestoreHostState() may unconditionally enables interrupts. */
6669	#error "VMM: Fix Me! Make VMXRestoreHostState() function to skip cli/sti."
6670	#else
6671	Assert(ASMIntAreEnabled());
6672	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6673	#endif
6674	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6675	}
6676
6677	/*
6678	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6679	* and mark the VMCS launch-state as "clear".
6680	*/
6681	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6682	return rc;
6683	}
6684
6685
6686	/**
6687	* Saves the host state in the VMCS host-state.
6688	* Sets up the VM-exit MSR-load area.
6689	*
6690	* The CPU state will be loaded from these fields on every successful VM-exit.
6691	*
6692	* @returns VBox status code.
6693	* @param pVM Pointer to the VM.
6694	* @param pVCpu Pointer to the VMCPU.
6695	*
6696	* @remarks No-long-jump zone!!!
6697	*/
6698	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6699	{
6700	AssertPtr(pVM);
6701	AssertPtr(pVCpu);
6702	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6703
6704	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6705
6706	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6707	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6708	return VINF_SUCCESS;
6709
6710	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6711	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6712
6713	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6714	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6715
6716	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6717	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6718
6719	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6720	return rc;
6721	}
6722
6723
6724	/**
6725	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6726	* loaded from these fields on every successful VM-entry.
6727	*
6728	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6729	* Sets up the VM-entry controls.
6730	* Sets up the appropriate VMX non-root function to execute guest code based on
6731	* the guest CPU mode.
6732	*
6733	* @returns VBox status code.
6734	* @param pVM Pointer to the VM.
6735	* @param pVCpu Pointer to the VMCPU.
6736	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6737	* out-of-sync. Make sure to update the required fields
6738	* before using them.
6739	*
6740	* @remarks No-long-jump zone!!!
6741	*/
6742	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6743	{
6744	AssertPtr(pVM);
6745	AssertPtr(pVCpu);
6746	AssertPtr(pMixedCtx);
6747	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6748
6749	#ifdef LOG_ENABLED
6750	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
6751	* probably not initialized yet? Anyway this will do for now. */
6752	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
6753	VMMR0LogFlushDisable(pVCpu);
6754	#endif
6755
6756	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6757
6758	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6759
6760	/* Determine real-on-v86 mode. */
6761	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6762	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6763	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6764	{
6765	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6766	}
6767
6768	/*
6769	* Load the guest-state into the VMCS.
6770	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6771	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6772	*/
6773	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6774	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6775
6776	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6777	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6778
6779	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6780	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6781
6782	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6783	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6784
6785	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6786	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6787	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6788
6789	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6790	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6791
6792	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6793	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6794
6795	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6796	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6797
6798	/* Must be done after hmR0VmxLoadGuestDebugRegs() as it may update eflags.TF for debugging purposes. */
6799	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6800	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6801
6802	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6803	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6804
6805	/* Clear any unused and reserved bits. */
6806	pVCpu->hm.s.fContextUseFlags &= ~( HM_CHANGED_GUEST_CR2
6807	\| HM_CHANGED_GUEST_MSR /* legacy */);
6808
6809	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6810	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p idCpu=%RU32 fContextUseFlags=%#RX32\n",
6811	pVM, pVCpu, pVCpu->idCpu, pVCpu->hm.s.fContextUseFlags));
6812
6813	#ifdef LOG_ENABLED
6814	/* Only reenable log-flushing if the caller has it enabled. */
6815	if (!fCallerDisabledLogFlush)
6816	VMMR0LogFlushEnable(pVCpu);
6817	#endif
6818
6819	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6820	return rc;
6821	}
6822
6823
6824	/**
6825	* Loads the guest state into the VMCS guest-state area.
6826	*
6827	* @returns VBox status code.
6828	* @param pVM Pointer to the VM.
6829	* @param pVCpu Pointer to the VMCPU.
6830	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6831	* out-of-sync. Make sure to update the required fields
6832	* before using them.
6833	*
6834	* @remarks No-long-jump zone!!!
6835	*/
6836	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6837	{
6838	/*
6839	* Avoid reloading the guest state on longjmp reentrants and do it lazily just before executing the guest.
6840	* This only helps when we get rescheduled more than once to a different host CPU on a longjmp trip before
6841	* finally executing guest code.
6842	*/
6843	return VINF_SUCCESS;
6844	}
6845
6846
6847	/**
6848	* Does the preparations before executing guest code in VT-x.
6849	*
6850	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6851	* recompiler. We must be cautious what we do here regarding committing
6852	* guest-state information into the the VMCS assuming we assuredly execute the
6853	* guest in VT-x. If we fall back to the recompiler after updating the VMCS and
6854	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6855	* that the recompiler can (and should) use them when it resumes guest
6856	* execution. Otherwise such operations must be done when we can no longer
6857	* exit to ring-3.
6858	*
6859	* @returns VBox status code (informational status codes included).
6860	* @retval VINF_SUCCESS if we can proceed with running the guest.
6861	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6862	* into the guest.
6863	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6864	*
6865	* @param pVM Pointer to the VM.
6866	* @param pVCpu Pointer to the VMCPU.
6867	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6868	* out-of-sync. Make sure to update the required fields
6869	* before using them.
6870	* @param pVmxTransient Pointer to the VMX transient structure.
6871	*
6872	* @remarks Called with preemption disabled.
6873	*/
6874	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6875	{
6876	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6877
6878	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6879	PGMRZDynMapFlushAutoSet(pVCpu);
6880	#endif
6881
6882	/* Check force flag actions that might require us to go back to ring-3. */
6883	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6884	if (rc != VINF_SUCCESS)
6885	return rc;
6886
6887	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6888	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6889	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6890	{
6891	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6892	RTGCPHYS GCPhysApicBase;
6893	GCPhysApicBase = pMixedCtx->msrApicBase;
6894	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6895
6896	/* Unalias any existing mapping. */
6897	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6898	AssertRCReturn(rc, rc);
6899
6900	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6901	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6902	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6903	AssertRCReturn(rc, rc);
6904
6905	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6906	}
6907
6908	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6909	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6910	pVmxTransient->uEFlags = ASMIntDisableFlags();
6911	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6912	{
6913	ASMSetFlags(pVmxTransient->uEFlags);
6914	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6915	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6916	return VINF_EM_RAW_INTERRUPT;
6917	}
6918	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6919	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6920	#endif
6921
6922	/*
6923	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6924	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6925	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6926	*/
6927	/** @todo Rework event evaluation and injection to be completely separate. */
6928	if (TRPMHasTrap(pVCpu))
6929	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
6930
6931	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6932	AssertRCReturn(rc, rc);
6933	return rc;
6934	}
6935
6936
6937	/**
6938	* Prepares to run guest code in VT-x and we've committed to doing so. This
6939	* means there is no backing out to ring-3 or anywhere else at this
6940	* point.
6941	*
6942	* @param pVM Pointer to the VM.
6943	* @param pVCpu Pointer to the VMCPU.
6944	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6945	* out-of-sync. Make sure to update the required fields
6946	* before using them.
6947	* @param pVmxTransient Pointer to the VMX transient structure.
6948	*
6949	* @remarks Called with preemption disabled.
6950	* @remarks No-long-jump zone!!!
6951	*/
6952	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6953	{
6954	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6955	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6956
6957	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6958	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6959	pVmxTransient->uEFlags = ASMIntDisableFlags();
6960	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6961	#endif
6962
6963	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6964	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6965	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6966	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6967	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6968	#endif
6969	int rc = VINF_SUCCESS;
6970	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6971	{
6972	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6973	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6974	}
6975	else if (pVCpu->hm.s.fContextUseFlags)
6976	{
6977	rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
6978	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6979	}
6980	AssertRC(rc);
6981	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6982
6983	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6984	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6985	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6986
6987	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6988	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6989	{
6990	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6991	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6992	}
6993
6994	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6995	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6996	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6997
6998	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6999
7000	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7001	to start executing. */
7002
7003	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7004	/*
7005	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7006	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7007	*/
7008	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7009	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7010	{
7011	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7012	uint64_t u64HostTscAux = 0;
7013	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7014	AssertRC(rc2);
7015	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7016	}
7017	#endif
7018	}
7019
7020
7021	/**
7022	* Performs some essential restoration of state after running guest code in
7023	* VT-x.
7024	*
7025	* @param pVM Pointer to the VM.
7026	* @param pVCpu Pointer to the VMCPU.
7027	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7028	* out-of-sync. Make sure to update the required fields
7029	* before using them.
7030	* @param pVmxTransient Pointer to the VMX transient structure.
7031	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7032	*
7033	* @remarks Called with interrupts disabled.
7034	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7035	* unconditionally when it is safe to do so.
7036	*/
7037	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7038	{
7039	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7040
7041	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7042	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7043	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7044	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7045	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7046
7047	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7048	{
7049	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7050	/* Restore host's TSC_AUX. */
7051	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7052	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7053	#endif
7054	/** @todo Find a way to fix hardcoding a guestimate. */
7055	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7056	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7057	}
7058
7059	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7060	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7061	Assert(!(ASMGetFlags() & X86_EFL_IF));
7062	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7063
7064	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
7065	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7066
7067	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7068	uint32_t uExitReason;
7069	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7070	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7071	AssertRC(rc);
7072	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7073	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7074
7075	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
7076	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7077
7078	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7079	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7080	{
7081	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7082	pVmxTransient->fVMEntryFailed));
7083	return;
7084	}
7085
7086	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7087	{
7088	/* Update the guest interruptibility-state from the VMCS. */
7089	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7090	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7091	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7092	AssertRC(rc);
7093	#endif
7094	/*
7095	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7096	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7097	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7098	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7099	*/
7100	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7101	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7102	{
7103	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7104	AssertRC(rc);
7105	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7106	}
7107	}
7108	}
7109
7110
7111	/**
7112	* Runs the guest code using VT-x.
7113	*
7114	* @returns VBox status code.
7115	* @param pVM Pointer to the VM.
7116	* @param pVCpu Pointer to the VMCPU.
7117	* @param pCtx Pointer to the guest-CPU context.
7118	*
7119	* @remarks Called with preemption disabled.
7120	*/
7121	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7122	{
7123	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7124	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7125
7126	VMXTRANSIENT VmxTransient;
7127	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7128	int rc = VERR_INTERNAL_ERROR_5;
7129	uint32_t cLoops = 0;
7130
7131	for (;; cLoops++)
7132	{
7133	Assert(!HMR0SuspendPending());
7134	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7135	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7136	(unsigned)RTMpCpuId(), cLoops));
7137
7138	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7139	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7140	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7141	if (rc != VINF_SUCCESS)
7142	break;
7143
7144	/*
7145	* No longjmps to ring-3 from this point on!!!
7146	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7147	* This also disables flushing of the R0-logger instance (if any).
7148	*/
7149	VMMRZCallRing3Disable(pVCpu);
7150	VMMRZCallRing3RemoveNotification(pVCpu);
7151	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7152
7153	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7154	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7155
7156	/*
7157	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7158	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7159	*/
7160	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7161	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7162	{
7163	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7164	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7165	return rc;
7166	}
7167
7168	/* Handle the VM-exit. */
7169	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7170	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7171	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7172	HMVMX_START_EXIT_DISPATCH_PROF();
7173	#ifdef HMVMX_USE_FUNCTION_TABLE
7174	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7175	#else
7176	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7177	#endif
7178	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7179	if (rc != VINF_SUCCESS)
7180	break;
7181	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7182	{
7183	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7184	rc = VINF_EM_RAW_INTERRUPT;
7185	break;
7186	}
7187	}
7188
7189	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7190	if (rc == VERR_EM_INTERPRETER)
7191	rc = VINF_EM_RAW_EMULATE_INSTR;
7192	else if (rc == VINF_EM_RESET)
7193	rc = VINF_EM_TRIPLE_FAULT;
7194	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7195	return rc;
7196	}
7197
7198
7199	#ifndef HMVMX_USE_FUNCTION_TABLE
7200	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7201	{
7202	int rc;
7203	switch (rcReason)
7204	{
7205	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7206	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7207	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7208	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7209	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7210	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7211	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7212	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7213	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7214	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7215	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7216	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7217	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7218	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7219	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7220	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7221	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7222	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7223	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7224	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7225	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7226	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7227	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7228	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7229	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7230	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7231	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7232	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7233	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7234	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7235	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7236	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7237	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7238
7239	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7240	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7241	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7242	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7243	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7244	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7245	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7246	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7247	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7248
7249	case VMX_EXIT_VMCALL:
7250	case VMX_EXIT_VMCLEAR:
7251	case VMX_EXIT_VMLAUNCH:
7252	case VMX_EXIT_VMPTRLD:
7253	case VMX_EXIT_VMPTRST:
7254	case VMX_EXIT_VMREAD:
7255	case VMX_EXIT_VMRESUME:
7256	case VMX_EXIT_VMWRITE:
7257	case VMX_EXIT_VMXOFF:
7258	case VMX_EXIT_VMXON:
7259	case VMX_EXIT_INVEPT:
7260	case VMX_EXIT_INVVPID:
7261	case VMX_EXIT_VMFUNC:
7262	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7263	break;
7264	default:
7265	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7266	break;
7267	}
7268	return rc;
7269	}
7270	#endif
7271
7272	#ifdef DEBUG
7273	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7274	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7275	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7276
7277	# define HMVMX_ASSERT_PREEMPT_CPUID() \
7278	do \
7279	{ \
7280	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7281	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7282	} while (0)
7283
7284	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7285	do { \
7286	AssertPtr(pVCpu); \
7287	AssertPtr(pMixedCtx); \
7288	AssertPtr(pVmxTransient); \
7289	Assert(pVmxTransient->fVMEntryFailed == false); \
7290	Assert(ASMIntAreEnabled()); \
7291	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7292	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7293	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7294	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7295	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7296	HMVMX_ASSERT_PREEMPT_CPUID(); \
7297	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7298	} while (0)
7299
7300	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7301	do { \
7302	Log4Func(("\n")); \
7303	} while(0)
7304	#else /* Release builds */
7305	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7306	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7307	#endif
7308
7309
7310	/**
7311	* Advances the guest RIP after reading it from the VMCS.
7312	*
7313	* @returns VBox status code.
7314	* @param pVCpu Pointer to the VMCPU.
7315	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7316	* out-of-sync. Make sure to update the required fields
7317	* before using them.
7318	* @param pVmxTransient Pointer to the VMX transient structure.
7319	*
7320	* @remarks No-long-jump zone!!!
7321	*/
7322	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7323	{
7324	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7325	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7326	AssertRCReturn(rc, rc);
7327
7328	pMixedCtx->rip += pVmxTransient->cbInstr;
7329	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7330	return rc;
7331	}
7332
7333
7334	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7335	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7336	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7337
7338	/** @name VM-exit handlers.
7339	* @{
7340	*/
7341
7342	/**
7343	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7344	*/
7345	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7346	{
7347	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7348	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7349	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
7350	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
7351	Assert(ASMIntAreEnabled());
7352	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
7353	return VINF_SUCCESS;
7354	#endif
7355	return VINF_EM_RAW_INTERRUPT;
7356	}
7357
7358
7359	/**
7360	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
7361	*/
7362	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7363	{
7364	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7365	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7366
7367	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7368	AssertRCReturn(rc, rc);
7369
7370	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7371	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7372	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7373	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
7374
7375	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7376	{
7377	/*
7378	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
7379	* anything we inject is not going to cause a VM-exit directly for the event being injected.
7380	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
7381	*
7382	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
7383	*/
7384	VMXDispatchHostNmi();
7385	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmi);
7386	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7387	return VINF_SUCCESS;
7388	}
7389
7390	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7391	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7392	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
7393	{
7394	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7395	return VINF_SUCCESS;
7396	}
7397	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7398	{
7399	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7400	return rc;
7401	}
7402
7403	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7404	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7405	switch (uIntrType)
7406	{
7407	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7408	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7409	/* no break */
7410	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7411	{
7412	switch (uVector)
7413	{
7414	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7415	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7416	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7417	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7418	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7419	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7420	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7421	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7422	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7423	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7424	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7425	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7426	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7427	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7428	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7429	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7430	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7431	#endif
7432	default:
7433	{
7434	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7435	AssertRCReturn(rc, rc);
7436
7437	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7438	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7439	{
7440	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7441	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7442	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7443	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7444	AssertRCReturn(rc, rc);
7445	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7446	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7447	0 /* GCPtrFaultAddress */);
7448	AssertRCReturn(rc, rc);
7449	}
7450	else
7451	{
7452	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7453	pVCpu->hm.s.u32HMError = uVector;
7454	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7455	}
7456	break;
7457	}
7458	}
7459	break;
7460	}
7461
7462	default:
7463	{
7464	pVCpu->hm.s.u32HMError = uExitIntrInfo;
7465	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7466	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7467	break;
7468	}
7469	}
7470	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7471	return rc;
7472	}
7473
7474
7475	/**
7476	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7477	*/
7478	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7479	{
7480	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7481
7482	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7483	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7484	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7485	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7486	AssertRCReturn(rc, rc);
7487
7488	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7489	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7490	return VINF_SUCCESS;
7491	}
7492
7493
7494	/**
7495	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7496	*/
7497	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7498	{
7499	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7500	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7501	pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
7502	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7503	}
7504
7505
7506	/**
7507	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7508	*/
7509	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7510	{
7511	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7512	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7513	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7514	}
7515
7516
7517	/**
7518	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7519	*/
7520	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7521	{
7522	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7523	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7524	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7525	}
7526
7527
7528	/**
7529	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7530	*/
7531	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7532	{
7533	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7534	PVM pVM = pVCpu->CTX_SUFF(pVM);
7535	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7536	if (RT_LIKELY(rc == VINF_SUCCESS))
7537	{
7538	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7539	Assert(pVmxTransient->cbInstr == 2);
7540	}
7541	else
7542	{
7543	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7544	rc = VERR_EM_INTERPRETER;
7545	}
7546	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7547	return rc;
7548	}
7549
7550
7551	/**
7552	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7553	*/
7554	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7555	{
7556	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7557	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7558	AssertRCReturn(rc, rc);
7559
7560	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7561	return VINF_EM_RAW_EMULATE_INSTR;
7562
7563	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7564	pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
7565	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7566	}
7567
7568
7569	/**
7570	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7571	*/
7572	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7573	{
7574	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7575	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7576	AssertRCReturn(rc, rc);
7577
7578	PVM pVM = pVCpu->CTX_SUFF(pVM);
7579	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7580	if (RT_LIKELY(rc == VINF_SUCCESS))
7581	{
7582	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7583	Assert(pVmxTransient->cbInstr == 2);
7584	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7585	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7586	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7587	}
7588	else
7589	{
7590	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7591	rc = VERR_EM_INTERPRETER;
7592	}
7593	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7594	return rc;
7595	}
7596
7597
7598	/**
7599	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7600	*/
7601	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7602	{
7603	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7604	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7605	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7606	AssertRCReturn(rc, rc);
7607
7608	PVM pVM = pVCpu->CTX_SUFF(pVM);
7609	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7610	if (RT_LIKELY(rc == VINF_SUCCESS))
7611	{
7612	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7613	Assert(pVmxTransient->cbInstr == 3);
7614	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7615	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7616	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7617	}
7618	else
7619	{
7620	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7621	rc = VERR_EM_INTERPRETER;
7622	}
7623	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7624	return rc;
7625	}
7626
7627
7628	/**
7629	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7630	*/
7631	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7632	{
7633	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7634	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7635	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7636	AssertRCReturn(rc, rc);
7637
7638	PVM pVM = pVCpu->CTX_SUFF(pVM);
7639	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7640	if (RT_LIKELY(rc == VINF_SUCCESS))
7641	{
7642	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7643	Assert(pVmxTransient->cbInstr == 2);
7644	}
7645	else
7646	{
7647	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7648	rc = VERR_EM_INTERPRETER;
7649	}
7650	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7651	return rc;
7652	}
7653
7654
7655	/**
7656	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7657	*/
7658	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7659	{
7660	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7661	PVM pVM = pVCpu->CTX_SUFF(pVM);
7662	Assert(!pVM->hm.s.fNestedPaging);
7663
7664	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7665	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7666	AssertRCReturn(rc, rc);
7667
7668	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7669	rc = VBOXSTRICTRC_VAL(rc2);
7670	if (RT_LIKELY(rc == VINF_SUCCESS))
7671	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7672	else
7673	{
7674	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
7675	pVmxTransient->uExitQualification, rc));
7676	}
7677	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7678	return rc;
7679	}
7680
7681
7682	/**
7683	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7684	*/
7685	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7686	{
7687	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7688	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7689	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7690	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7691	AssertRCReturn(rc, rc);
7692
7693	PVM pVM = pVCpu->CTX_SUFF(pVM);
7694	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7695	if (RT_LIKELY(rc == VINF_SUCCESS))
7696	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7697	else
7698	{
7699	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7700	rc = VERR_EM_INTERPRETER;
7701	}
7702	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7703	return rc;
7704	}
7705
7706
7707	/**
7708	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7709	*/
7710	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7711	{
7712	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7713	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7714	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7715	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7716	AssertRCReturn(rc, rc);
7717
7718	PVM pVM = pVCpu->CTX_SUFF(pVM);
7719	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7720	rc = VBOXSTRICTRC_VAL(rc2);
7721	if (RT_LIKELY( rc == VINF_SUCCESS
7722	\|\| rc == VINF_EM_HALT))
7723	{
7724	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7725	AssertRCReturn(rc3, rc3);
7726
7727	if ( rc == VINF_EM_HALT
7728	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7729	{
7730	rc = VINF_SUCCESS;
7731	}
7732	}
7733	else
7734	{
7735	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7736	rc = VERR_EM_INTERPRETER;
7737	}
7738	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7739	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7740	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7741	return rc;
7742	}
7743
7744
7745	/**
7746	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7747	*/
7748	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7749	{
7750	/*
7751	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7752	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7753	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
7754	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7755	*/
7756	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7757	pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
7758	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7759	}
7760
7761
7762	/**
7763	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7764	*/
7765	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7766	{
7767	/*
7768	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7769	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
7770	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
7771	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7772	*/
7773	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7774	pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
7775	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7776	}
7777
7778
7779	/**
7780	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7781	*/
7782	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7783	{
7784	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7785	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7786	pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
7787	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7788	}
7789
7790
7791	/**
7792	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7793	*/
7794	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7795	{
7796	/*
7797	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7798	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7799	* See Intel spec. 25.3 "Other Causes of VM-exits".
7800	*/
7801	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7802	pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
7803	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7804	}
7805
7806
7807	/**
7808	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7809	* VM-exit.
7810	*/
7811	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7812	{
7813	/*
7814	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
7815	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
7816	*
7817	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
7818	* See Intel spec. "23.8 Restrictions on VMX operation".
7819	*/
7820	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7821	return VINF_SUCCESS;
7822	}
7823
7824
7825	/**
7826	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7827	* VM-exit.
7828	*/
7829	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7830	{
7831	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7832	return VINF_EM_RESET;
7833	}
7834
7835
7836	/**
7837	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7838	*/
7839	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7840	{
7841	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7842	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7843	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7844	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7845	AssertRCReturn(rc, rc);
7846
7847	pMixedCtx->rip++;
7848	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7849	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7850	rc = VINF_SUCCESS;
7851	else
7852	rc = VINF_EM_HALT;
7853
7854	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7855	return rc;
7856	}
7857
7858
7859	/**
7860	* VM-exit handler for instructions that result in a #UD exception delivered to
7861	* the guest.
7862	*/
7863	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7864	{
7865	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7866	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7867	return VINF_SUCCESS;
7868	}
7869
7870
7871	/**
7872	* VM-exit handler for expiry of the VMX preemption timer.
7873	*/
7874	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7875	{
7876	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7877
7878	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7879	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7880
7881	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7882	PVM pVM = pVCpu->CTX_SUFF(pVM);
7883	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7884	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7885	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7886	}
7887
7888
7889	/**
7890	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7891	*/
7892	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7893	{
7894	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7895
7896	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7897	/** @todo check if XSETBV is supported by the recompiler. */
7898	return VERR_EM_INTERPRETER;
7899	}
7900
7901
7902	/**
7903	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7904	*/
7905	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7906	{
7907	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7908
7909	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7910	/** @todo implement EMInterpretInvpcid() */
7911	return VERR_EM_INTERPRETER;
7912	}
7913
7914
7915	/**
7916	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7917	* Error VM-exit.
7918	*/
7919	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7920	{
7921	uint32_t uIntrState;
7922	HMVMXHCUINTREG uHCReg;
7923	uint64_t u64Val;
7924	uint32_t u32Val;
7925
7926	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7927	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7928	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7929	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7930	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7931	AssertRCReturn(rc, rc);
7932
7933	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7934	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7935	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7936	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7937
7938	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7939	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7940	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7941	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7942	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7943	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7944	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7945	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7946	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7947	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7948	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7949	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7950
7951	PVM pVM = pVCpu->CTX_SUFF(pVM);
7952	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7953
7954	return VERR_VMX_INVALID_GUEST_STATE;
7955	}
7956
7957
7958	/**
7959	* VM-exit handler for VM-entry failure due to an MSR-load
7960	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7961	*/
7962	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7963	{
7964	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7965	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7966	}
7967
7968
7969	/**
7970	* VM-exit handler for VM-entry failure due to a machine-check event
7971	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7972	*/
7973	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7974	{
7975	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7976	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7977	}
7978
7979
7980	/**
7981	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7982	* theory.
7983	*/
7984	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7985	{
7986	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7987	return VERR_VMX_UNDEFINED_EXIT_CODE;
7988	}
7989
7990
7991	/**
7992	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7993	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7994	* Conditional VM-exit.
7995	*/
7996	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7997	{
7998	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7999
8000	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
8001	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
8002	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
8003	return VERR_EM_INTERPRETER;
8004	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8005	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8006	}
8007
8008
8009	/**
8010	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
8011	*/
8012	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8013	{
8014	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8015
8016	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
8017	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
8018	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
8019	return VERR_EM_INTERPRETER;
8020	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8021	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8022	}
8023
8024
8025	/**
8026	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
8027	*/
8028	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8029	{
8030	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8031
8032	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
8033	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8034	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8035	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8036	AssertRCReturn(rc, rc);
8037
8038	PVM pVM = pVCpu->CTX_SUFF(pVM);
8039	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8040	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
8041	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
8042	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
8043
8044	if (RT_LIKELY(rc == VINF_SUCCESS))
8045	{
8046	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8047	Assert(pVmxTransient->cbInstr == 2);
8048	}
8049	return rc;
8050	}
8051
8052
8053	/**
8054	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
8055	*/
8056	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8057	{
8058	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8059	PVM pVM = pVCpu->CTX_SUFF(pVM);
8060	int rc = VINF_SUCCESS;
8061
8062	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
8063	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8064	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8065	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8066	AssertRCReturn(rc, rc);
8067	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
8068
8069	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8070	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
8071	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
8072
8073	if (RT_LIKELY(rc == VINF_SUCCESS))
8074	{
8075	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8076
8077	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
8078	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
8079	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
8080	{
8081	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
8082	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
8083	EMInterpretWrmsr() changes it. */
8084	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8085	}
8086	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
8087	{
8088	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8089	AssertRCReturn(rc, rc);
8090	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
8091	}
8092	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
8093	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8094
8095	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
8096	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8097	{
8098	switch (pMixedCtx->ecx)
8099	{
8100	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
8101	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
8102	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
8103	case MSR_K8_FS_BASE: /* no break */
8104	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
8105	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
8106	}
8107	}
8108	#ifdef VBOX_STRICT
8109	else
8110	{
8111	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
8112	switch (pMixedCtx->ecx)
8113	{
8114	case MSR_IA32_SYSENTER_CS:
8115	case MSR_IA32_SYSENTER_EIP:
8116	case MSR_IA32_SYSENTER_ESP:
8117	case MSR_K8_FS_BASE:
8118	case MSR_K8_GS_BASE:
8119	{
8120	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
8121	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8122	}
8123
8124	case MSR_K8_LSTAR:
8125	case MSR_K6_STAR:
8126	case MSR_K8_SF_MASK:
8127	case MSR_K8_TSC_AUX:
8128	case MSR_K8_KERNEL_GS_BASE:
8129	{
8130	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
8131	pMixedCtx->ecx));
8132	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8133	}
8134	}
8135	}
8136	#endif /* VBOX_STRICT */
8137	}
8138	return rc;
8139	}
8140
8141
8142	/**
8143	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
8144	*/
8145	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8146	{
8147	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8148
8149	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
8150	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
8151	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
8152	return VERR_EM_INTERPRETER;
8153	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8154	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8155	}
8156
8157
8158	/**
8159	* VM-exit handler for when the TPR value is lowered below the specified
8160	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
8161	*/
8162	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8163	{
8164	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8165	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
8166
8167	/*
8168	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
8169	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
8170	* resume guest execution.
8171	*/
8172	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8173	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
8174	return VINF_SUCCESS;
8175	}
8176
8177
8178	/**
8179	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
8180	* VM-exit.
8181	*
8182	* @retval VINF_SUCCESS when guest execution can continue.
8183	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
8184	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
8185	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
8186	* recompiler.
8187	*/
8188	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8189	{
8190	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8191	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
8192	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8193	AssertRCReturn(rc, rc);
8194
8195	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
8196	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
8197	PVM pVM = pVCpu->CTX_SUFF(pVM);
8198	switch (uAccessType)
8199	{
8200	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
8201	{
8202	#if 0
8203	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
8204	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8205	#else
8206	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8207	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8208	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8209	#endif
8210	AssertRCReturn(rc, rc);
8211
8212	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8213	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
8214	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
8215	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
8216
8217	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
8218	{
8219	case 0: /* CR0 */
8220	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
8221	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8222	break;
8223	case 2: /* C2 **/
8224	/* Nothing to do here, CR2 it's not part of the VMCS. */
8225	break;
8226	case 3: /* CR3 */
8227	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
8228	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
8229	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
8230	break;
8231	case 4: /* CR4 */
8232	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
8233	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
8234	break;
8235	case 8: /* CR8 */
8236	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8237	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
8238	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8239	break;
8240	default:
8241	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
8242	break;
8243	}
8244
8245	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8246	break;
8247	}
8248
8249	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
8250	{
8251	/* EMInterpretCRxRead() requires EFER MSR, CS. */
8252	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8253	AssertRCReturn(rc, rc);
8254	Assert( !pVM->hm.s.fNestedPaging
8255	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
8256	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
8257
8258	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8259	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
8260	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8261
8262	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8263	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
8264	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
8265	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8266	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8267	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
8268	break;
8269	}
8270
8271	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
8272	{
8273	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8274	AssertRCReturn(rc, rc);
8275	rc = EMInterpretCLTS(pVM, pVCpu);
8276	AssertRCReturn(rc, rc);
8277	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8278	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
8279	Log4(("CRX CLTS write rc=%d\n", rc));
8280	break;
8281	}
8282
8283	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
8284	{
8285	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8286	AssertRCReturn(rc, rc);
8287	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
8288	if (RT_LIKELY(rc == VINF_SUCCESS))
8289	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8290	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
8291	Log4(("CRX LMSW write rc=%d\n", rc));
8292	break;
8293	}
8294
8295	default:
8296	{
8297	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
8298	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8299	}
8300	}
8301
8302	/* Validate possible error codes. */
8303	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
8304	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8305	if (RT_SUCCESS(rc))
8306	{
8307	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8308	AssertRCReturn(rc2, rc2);
8309	}
8310
8311	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8312	return rc;
8313	}
8314
8315
8316	/**
8317	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8318	* VM-exit.
8319	*/
8320	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8321	{
8322	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8323	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8324
8325	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8326	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8327	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8328	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8329	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8330	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8331	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8332	AssertRCReturn(rc, rc);
8333
8334	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8335
8336	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8337	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8338	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8339	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8340	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8341	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
8342	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
8343
8344	/* I/O operation lookup arrays. */
8345	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
8346	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8347
8348	const uint32_t cbSize = s_aIOSize[uIOWidth];
8349	const uint32_t cbInstr = pVmxTransient->cbInstr;
8350	PVM pVM = pVCpu->CTX_SUFF(pVM);
8351	if (fIOString)
8352	{
8353	/* INS/OUTS - I/O String instruction. */
8354	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8355	/** @todo for now manually disassemble later optimize by getting the fields from
8356	* the VMCS. VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8357	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8358	* segment prefix info. */
8359	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8360	if (RT_SUCCESS(rc))
8361	{
8362	if (fIOWrite)
8363	{
8364	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8365	(DISCPUMODE)pDis->uAddrMode, cbSize);
8366	rc = VBOXSTRICTRC_VAL(rc2);
8367	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8368	}
8369	else
8370	{
8371	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8372	(DISCPUMODE)pDis->uAddrMode, cbSize);
8373	rc = VBOXSTRICTRC_VAL(rc2);
8374	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8375	}
8376	}
8377	else
8378	{
8379	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8380	rc = VINF_EM_RAW_EMULATE_INSTR;
8381	}
8382	}
8383	else
8384	{
8385	/* IN/OUT - I/O instruction. */
8386	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8387	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8388	if (fIOWrite)
8389	{
8390	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8391	rc = VBOXSTRICTRC_VAL(rc2);
8392	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8393	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8394	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8395	}
8396	else
8397	{
8398	uint32_t u32Result = 0;
8399	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8400	rc = VBOXSTRICTRC_VAL(rc2);
8401	if (IOM_SUCCESS(rc))
8402	{
8403	/* Save result of I/O IN instr. in AL/AX/EAX. */
8404	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8405	}
8406	else if (rc == VINF_IOM_R3_IOPORT_READ)
8407	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8408	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8409	}
8410	}
8411
8412	if (IOM_SUCCESS(rc))
8413	{
8414	pMixedCtx->rip += cbInstr;
8415	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8416	if (RT_LIKELY(rc == VINF_SUCCESS))
8417	{
8418	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8419	AssertRCReturn(rc, rc);
8420
8421	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8422	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8423	{
8424	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8425	for (unsigned i = 0; i < 4; i++)
8426	{
8427	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8428	if ( ( uIOPort >= pMixedCtx->dr[i]
8429	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8430	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8431	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8432	{
8433	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8434	uint64_t uDR6 = ASMGetDR6();
8435
8436	/* Clear all breakpoint status flags and set the one we just hit. */
8437	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8438	uDR6 \|= (uint64_t)RT_BIT(i);
8439
8440	/*
8441	* Note: AMD64 Architecture Programmer's Manual 13.1:
8442	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8443	* be cleared by software after the contents have been read.
8444	*/
8445	ASMSetDR6(uDR6);
8446
8447	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8448	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8449
8450	/* Paranoia. */
8451	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits MBZ. */
8452	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8453	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8454
8455	/* Resync DR7 */
8456	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8457	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8458
8459	/* Set #DB to be injected into the VM and continue guest execution. */
8460	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8461	break;
8462	}
8463	}
8464	}
8465	}
8466	}
8467
8468	#ifdef DEBUG
8469	if (rc == VINF_IOM_R3_IOPORT_READ)
8470	Assert(!fIOWrite);
8471	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8472	Assert(fIOWrite);
8473	else
8474	{
8475	AssertMsg( RT_FAILURE(rc)
8476	\|\| rc == VINF_SUCCESS
8477	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8478	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8479	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8480	}
8481	#endif
8482
8483	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8484	return rc;
8485	}
8486
8487
8488	/**
8489	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8490	* VM-exit.
8491	*/
8492	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8493	{
8494	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8495
8496	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8497	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8498	AssertRCReturn(rc, rc);
8499	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8500	{
8501	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8502	AssertRCReturn(rc, rc);
8503	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8504	{
8505	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8506
8507	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8508	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8509	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8510	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8511	{
8512	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8513	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8514
8515	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8516	Assert(!pVCpu->hm.s.Event.fPending);
8517	pVCpu->hm.s.Event.fPending = true;
8518	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8519	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8520	AssertRCReturn(rc, rc);
8521	if (fErrorCodeValid)
8522	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8523	else
8524	pVCpu->hm.s.Event.u32ErrCode = 0;
8525	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8526	&& uVector == X86_XCPT_PF)
8527	{
8528	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8529	}
8530
8531	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8532	}
8533	}
8534	}
8535
8536	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8537	* emulation. */
8538	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8539	return VERR_EM_INTERPRETER;
8540	}
8541
8542
8543	/**
8544	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8545	*/
8546	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8547	{
8548	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8549	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8550	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8551	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8552	AssertRCReturn(rc, rc);
8553	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8554	return VINF_EM_DBG_STOP;
8555	}
8556
8557
8558	/**
8559	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8560	*/
8561	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8562	{
8563	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8564
8565	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8566	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8567	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8568	return VINF_SUCCESS;
8569	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8570	return rc;
8571
8572	#if 0
8573	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8574	* just sync the whole thing. */
8575	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8576	#else
8577	/* Aggressive state sync. for now. */
8578	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8579	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8580	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8581	#endif
8582	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8583	AssertRCReturn(rc, rc);
8584
8585	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8586	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8587	switch (uAccessType)
8588	{
8589	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8590	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8591	{
8592	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8593	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8594	{
8595	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8596	}
8597
8598	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8599	GCPhys &= PAGE_BASE_GC_MASK;
8600	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8601	PVM pVM = pVCpu->CTX_SUFF(pVM);
8602	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
8603	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8604
8605	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8606	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8607	CPUMCTX2CORE(pMixedCtx), GCPhys);
8608	rc = VBOXSTRICTRC_VAL(rc2);
8609	Log4(("ApicAccess rc=%d\n", rc));
8610	if ( rc == VINF_SUCCESS
8611	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8612	\|\| rc == VERR_PAGE_NOT_PRESENT)
8613	{
8614	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8615	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8616	rc = VINF_SUCCESS;
8617	}
8618	break;
8619	}
8620
8621	default:
8622	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
8623	rc = VINF_EM_RAW_EMULATE_INSTR;
8624	break;
8625	}
8626
8627	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8628	return rc;
8629	}
8630
8631
8632	/**
8633	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8634	* VM-exit.
8635	*/
8636	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8637	{
8638	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8639
8640	/* We should -not- get this VM-exit if the guest is debugging. */
8641	if (CPUMIsGuestDebugStateActive(pVCpu))
8642	{
8643	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8644	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8645	}
8646
8647	int rc = VERR_INTERNAL_ERROR_5;
8648	if ( !DBGFIsStepping(pVCpu)
8649	&& !CPUMIsHyperDebugStateActive(pVCpu))
8650	{
8651	/* Don't intercept MOV DRx. */
8652	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8653	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8654	AssertRCReturn(rc, rc);
8655
8656	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8657	PVM pVM = pVCpu->CTX_SUFF(pVM);
8658	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8659	AssertRC(rc);
8660	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8661
8662	#ifdef VBOX_WITH_STATISTICS
8663	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8664	AssertRCReturn(rc, rc);
8665	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8666	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8667	else
8668	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8669	#endif
8670	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8671	return VINF_SUCCESS;
8672	}
8673
8674	/*
8675	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8676	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8677	*/
8678	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8679	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8680	AssertRCReturn(rc, rc);
8681
8682	PVM pVM = pVCpu->CTX_SUFF(pVM);
8683	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8684	{
8685	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8686	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8687	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8688	if (RT_SUCCESS(rc))
8689	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8690	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8691	}
8692	else
8693	{
8694	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8695	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8696	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8697	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8698	}
8699
8700	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8701	if (RT_SUCCESS(rc))
8702	{
8703	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8704	AssertRCReturn(rc2, rc2);
8705	}
8706	return rc;
8707	}
8708
8709
8710	/**
8711	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8712	* Conditional VM-exit.
8713	*/
8714	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8715	{
8716	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8717	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8718
8719	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8720	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8721	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8722	return VINF_SUCCESS;
8723	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8724	return rc;
8725
8726	RTGCPHYS GCPhys = 0;
8727	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8728
8729	#if 0
8730	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8731	#else
8732	/* Aggressive state sync. for now. */
8733	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8734	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8735	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8736	#endif
8737	AssertRCReturn(rc, rc);
8738
8739	/*
8740	* If we succeed, resume guest execution.
8741	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8742	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8743	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8744	* weird case. See @bugref{6043}.
8745	*/
8746	PVM pVM = pVCpu->CTX_SUFF(pVM);
8747	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8748	rc = VBOXSTRICTRC_VAL(rc2);
8749	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8750	if ( rc == VINF_SUCCESS
8751	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8752	\|\| rc == VERR_PAGE_NOT_PRESENT)
8753	{
8754	/* Successfully handled MMIO operation. */
8755	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8756	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8757	rc = VINF_SUCCESS;
8758	}
8759	return rc;
8760	}
8761
8762
8763	/**
8764	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8765	* VM-exit.
8766	*/
8767	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8768	{
8769	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8770	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8771
8772	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8773	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8774	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8775	return VINF_SUCCESS;
8776	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8777	return rc;
8778
8779	RTGCPHYS GCPhys = 0;
8780	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8781	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8782	#if 0
8783	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8784	#else
8785	/* Aggressive state sync. for now. */
8786	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8787	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8788	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8789	#endif
8790	AssertRCReturn(rc, rc);
8791
8792	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8793	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
8794
8795	RTGCUINT uErrorCode = 0;
8796	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8797	uErrorCode \|= X86_TRAP_PF_ID;
8798	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8799	uErrorCode \|= X86_TRAP_PF_RW;
8800	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8801	uErrorCode \|= X86_TRAP_PF_P;
8802
8803	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8804
8805	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
8806	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8807
8808	/* Handle the pagefault trap for the nested shadow table. */
8809	PVM pVM = pVCpu->CTX_SUFF(pVM);
8810	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8811	TRPMResetTrap(pVCpu);
8812
8813	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8814	if ( rc == VINF_SUCCESS
8815	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8816	\|\| rc == VERR_PAGE_NOT_PRESENT)
8817	{
8818	/* Successfully synced our nested page tables. */
8819	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8820	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8821	return VINF_SUCCESS;
8822	}
8823
8824	Log4(("EPT return to ring-3 rc=%d\n"));
8825	return rc;
8826	}
8827
8828	/** @} */
8829
8830	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8831	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8832	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8833
8834	/** @name VM-exit exception handlers.
8835	* @{
8836	*/
8837
8838	/**
8839	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8840	*/
8841	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8842	{
8843	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8844	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8845
8846	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8847	AssertRCReturn(rc, rc);
8848
8849	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8850	{
8851	/* Old-style FPU error reporting needs some extra work. */
8852	/** @todo don't fall back to the recompiler, but do it manually. */
8853	return VERR_EM_INTERPRETER;
8854	}
8855
8856	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8857	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8858	return rc;
8859	}
8860
8861
8862	/**
8863	* VM-exit exception handler for #BP (Breakpoint exception).
8864	*/
8865	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8866	{
8867	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8868	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8869
8870	/** @todo Try optimize this by not saving the entire guest state unless
8871	* really needed. */
8872	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8873	AssertRCReturn(rc, rc);
8874
8875	PVM pVM = pVCpu->CTX_SUFF(pVM);
8876	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8877	if (rc == VINF_EM_RAW_GUEST_TRAP)
8878	{
8879	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8880	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8881	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8882	AssertRCReturn(rc, rc);
8883
8884	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8885	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8886	}
8887
8888	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8889	return rc;
8890	}
8891
8892
8893	/**
8894	* VM-exit exception handler for #DB (Debug exception).
8895	*/
8896	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8897	{
8898	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8899	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8900
8901	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8902	AssertRCReturn(rc, rc);
8903
8904	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8905	uint64_t uDR6 = X86_DR6_INIT_VAL;
8906	uDR6 \|= (pVmxTransient->uExitQualification
8907	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8908	PVM pVM = pVCpu->CTX_SUFF(pVM);
8909	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8910	if (rc == VINF_EM_RAW_GUEST_TRAP)
8911	{
8912	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8913	pMixedCtx->dr[6] = uDR6;
8914
8915	if (CPUMIsGuestDebugStateActive(pVCpu))
8916	ASMSetDR6(pMixedCtx->dr[6]);
8917
8918	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8919
8920	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8921	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8922
8923	/* Paranoia. */
8924	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits MBZ. */
8925	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8926	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8927
8928	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
8929	AssertRCReturn(rc,rc);
8930
8931	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8932	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8933	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8934	AssertRCReturn(rc2, rc2);
8935	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8936	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8937	rc = VINF_SUCCESS;
8938	}
8939
8940	return rc;
8941	}
8942
8943
8944	/**
8945	* VM-exit exception handler for #NM (Device-not-available exception: floating
8946	* point exception).
8947	*/
8948	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8949	{
8950	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8951
8952	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8953	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8954	#endif
8955
8956	/* We require CR0 and EFER. EFER is always up-to-date. */
8957	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8958	AssertRCReturn(rc, rc);
8959
8960	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8961	PVM pVM = pVCpu->CTX_SUFF(pVM);
8962	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8963	if (rc == VINF_SUCCESS)
8964	{
8965	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8966	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8967	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8968	return VINF_SUCCESS;
8969	}
8970
8971	/* Forward #NM to the guest. */
8972	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8973	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8974	AssertRCReturn(rc, rc);
8975	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8976	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8977	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8978	return rc;
8979	}
8980
8981
8982	/**
8983	* VM-exit exception handler for #GP (General-protection exception).
8984	*
8985	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8986	*/
8987	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8988	{
8989	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8990	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8991
8992	int rc = VERR_INTERNAL_ERROR_5;
8993	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8994	{
8995	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8996	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8997	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8998	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8999	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9000	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9001	AssertRCReturn(rc, rc);
9002	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
9003	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
9004	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9005	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9006	return rc;
9007	#else
9008	/* We don't intercept #GP. */
9009	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
9010	return VERR_VMX_UNEXPECTED_EXCEPTION;
9011	#endif
9012	}
9013
9014	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9015	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
9016
9017	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
9018	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9019	AssertRCReturn(rc, rc);
9020
9021	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9022	uint32_t cbOp = 0;
9023	PVM pVM = pVCpu->CTX_SUFF(pVM);
9024	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
9025	if (RT_SUCCESS(rc))
9026	{
9027	rc = VINF_SUCCESS;
9028	Assert(cbOp == pDis->cbInstr);
9029	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
9030	switch (pDis->pCurInstr->uOpcode)
9031	{
9032	case OP_CLI:
9033	{
9034	pMixedCtx->eflags.Bits.u1IF = 0;
9035	pMixedCtx->rip += pDis->cbInstr;
9036	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
9037	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
9038	break;
9039	}
9040
9041	case OP_STI:
9042	{
9043	pMixedCtx->eflags.Bits.u1IF = 1;
9044	pMixedCtx->rip += pDis->cbInstr;
9045	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
9046	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9047	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
9048	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
9049	break;
9050	}
9051
9052	case OP_HLT:
9053	{
9054	rc = VINF_EM_HALT;
9055	pMixedCtx->rip += pDis->cbInstr;
9056	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9057	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9058	break;
9059	}
9060
9061	case OP_POPF:
9062	{
9063	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
9064	uint32_t cbParm = 0;
9065	uint32_t uMask = 0;
9066	if (pDis->fPrefix & DISPREFIX_OPSIZE)
9067	{
9068	cbParm = 4;
9069	uMask = 0xffffffff;
9070	}
9071	else
9072	{
9073	cbParm = 2;
9074	uMask = 0xffff;
9075	}
9076
9077	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
9078	RTGCPTR GCPtrStack = 0;
9079	X86EFLAGS uEflags;
9080	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9081	&GCPtrStack);
9082	if (RT_SUCCESS(rc))
9083	{
9084	Assert(sizeof(uEflags.u32) >= cbParm);
9085	uEflags.u32 = 0;
9086	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
9087	}
9088	if (RT_FAILURE(rc))
9089	{
9090	rc = VERR_EM_INTERPRETER;
9091	break;
9092	}
9093	Log4(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
9094	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9095	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
9096	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
9097	pMixedCtx->eflags.Bits.u1RF = 0;
9098	pMixedCtx->esp += cbParm;
9099	pMixedCtx->esp &= uMask;
9100	pMixedCtx->rip += pDis->cbInstr;
9101	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
9102	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
9103	break;
9104	}
9105
9106	case OP_PUSHF:
9107	{
9108	uint32_t cbParm = 0;
9109	uint32_t uMask = 0;
9110	if (pDis->fPrefix & DISPREFIX_OPSIZE)
9111	{
9112	cbParm = 4;
9113	uMask = 0xffffffff;
9114	}
9115	else
9116	{
9117	cbParm = 2;
9118	uMask = 0xffff;
9119	}
9120
9121	/* Get the stack pointer & push the contents of eflags onto the stack. */
9122	RTGCPTR GCPtrStack = 0;
9123	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
9124	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
9125	if (RT_FAILURE(rc))
9126	{
9127	rc = VERR_EM_INTERPRETER;
9128	break;
9129	}
9130	X86EFLAGS uEflags;
9131	uEflags = pMixedCtx->eflags;
9132	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
9133	uEflags.Bits.u1RF = 0;
9134	uEflags.Bits.u1VM = 0;
9135
9136	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
9137	if (RT_FAILURE(rc))
9138	{
9139	rc = VERR_EM_INTERPRETER;
9140	break;
9141	}
9142	Log4(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
9143	pMixedCtx->esp -= cbParm;
9144	pMixedCtx->esp &= uMask;
9145	pMixedCtx->rip += pDis->cbInstr;
9146	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
9147	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
9148	break;
9149	}
9150
9151	case OP_IRET:
9152	{
9153	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
9154	* instruction reference. */
9155	RTGCPTR GCPtrStack = 0;
9156	uint32_t uMask = 0xffff;
9157	uint16_t aIretFrame[3];
9158	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
9159	{
9160	rc = VERR_EM_INTERPRETER;
9161	break;
9162	}
9163	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9164	&GCPtrStack);
9165	if (RT_SUCCESS(rc))
9166	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
9167	if (RT_FAILURE(rc))
9168	{
9169	rc = VERR_EM_INTERPRETER;
9170	break;
9171	}
9172	pMixedCtx->eip = 0;
9173	pMixedCtx->ip = aIretFrame[0];
9174	pMixedCtx->cs.Sel = aIretFrame[1];
9175	pMixedCtx->cs.ValidSel = aIretFrame[1];
9176	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
9177	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9178	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
9179	pMixedCtx->sp += sizeof(aIretFrame);
9180	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
9181	\| HM_CHANGED_GUEST_RFLAGS;
9182	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
9183	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
9184	break;
9185	}
9186
9187	case OP_INT:
9188	{
9189	uint16_t uVector = pDis->Param1.uValue & 0xff;
9190	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
9191	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9192	break;
9193	}
9194
9195	case OP_INTO:
9196	{
9197	if (pMixedCtx->eflags.Bits.u1OF)
9198	{
9199	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
9200	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9201	}
9202	break;
9203	}
9204
9205	default:
9206	{
9207	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
9208	EMCODETYPE_SUPERVISOR);
9209	rc = VBOXSTRICTRC_VAL(rc2);
9210	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
9211	Log4(("#GP rc=%Rrc\n", rc));
9212	break;
9213	}
9214	}
9215	}
9216	else
9217	rc = VERR_EM_INTERPRETER;
9218
9219	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
9220	("#GP Unexpected rc=%Rrc\n", rc));
9221	return rc;
9222	}
9223
9224
9225	/**
9226	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
9227	* the exception reported in the VMX transient structure back into the VM.
9228	*
9229	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
9230	* up-to-date.
9231	*/
9232	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9233	{
9234	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9235
9236	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
9237	hmR0VmxCheckExitDueToEventDelivery(). */
9238	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9239	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9240	AssertRCReturn(rc, rc);
9241	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
9242
9243	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9244	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9245	return VINF_SUCCESS;
9246	}
9247
9248
9249	/**
9250	* VM-exit exception handler for #PF (Page-fault exception).
9251	*/
9252	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9253	{
9254	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9255	PVM pVM = pVCpu->CTX_SUFF(pVM);
9256	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9257	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9258	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9259	AssertRCReturn(rc, rc);
9260
9261	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
9262	if (pVM->hm.s.fNestedPaging)
9263	{
9264	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
9265	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
9266	{
9267	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9268	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9269	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
9270	}
9271	else
9272	{
9273	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9274	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9275	Log4(("Pending #DF due to vectoring #PF. NP\n"));
9276	}
9277	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9278	return rc;
9279	}
9280	#else
9281	Assert(!pVM->hm.s.fNestedPaging);
9282	#endif
9283
9284	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9285	AssertRCReturn(rc, rc);
9286
9287	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
9288	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9289
9290	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9291	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9292	(RTGCPTR)pVmxTransient->uExitQualification);
9293
9294	Log4(("#PF: rc=%Rrc\n", rc));
9295	if (rc == VINF_SUCCESS)
9296	{
9297	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9298	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9299	* memory? We don't update the whole state here... */
9300	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9301	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9302	TRPMResetTrap(pVCpu);
9303	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9304	return rc;
9305	}
9306	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9307	{
9308	if (!pVmxTransient->fVectoringPF)
9309	{
9310	/* It's a guest page fault and needs to be reflected to the guest. */
9311	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9312	TRPMResetTrap(pVCpu);
9313	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9314	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9315	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9316	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9317	}
9318	else
9319	{
9320	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9321	TRPMResetTrap(pVCpu);
9322	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
9323	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9324	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
9325	}
9326
9327	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9328	return VINF_SUCCESS;
9329	}
9330
9331	TRPMResetTrap(pVCpu);
9332	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9333	return rc;
9334	}
9335
9336	/** @} */
9337

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 47247

Download in other formats: