HMVMXR0.cpp@ 46286

Last change on this file since 46286 was 46286, checked in by vboxsync, 12 years ago
VMM/HMVMXR0: Avoid saving/restoring EFER whenever possible on every VM-entry/VM-exit.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 385.1 KB

Line
1	/* $Id: HMVMXR0.cpp 46286 2013-05-27 13:44:19Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/**
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126
127	/**
128	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
129	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
130	* we have Nested Paging support.
131	*/
132	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
133	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
134	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
135	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
136	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
137	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
138	\| RT_BIT(X86_XCPT_XF))
139
140	/**
141	* Exception bitmap mask for all contributory exceptions.
142	*/
143	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
144	\| RT_BIT(X86_XCPT_DE))
145
146	/** Maximum VM-instruction error number. */
147	#define HMVMX_INSTR_ERROR_MAX 28
148
149	/** Profiling macro. */
150	#ifdef HM_PROFILE_EXIT_DISPATCH
151	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
152	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
153	#else
154	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
156	#endif
157
158
159	/*******************************************************************************
160	* Structures and Typedefs *
161	*******************************************************************************/
162	/**
163	* A state structure for holding miscellaneous information across
164	* VMX non-root operation and restored after the transition.
165	*/
166	typedef struct VMXTRANSIENT
167	{
168	/** The host's rflags/eflags. */
169	RTCCUINTREG uEFlags;
170	#if HC_ARCH_BITS == 32
171	uint32_t u32Alignment0;
172	#endif
173	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
174	uint64_t u64LStarMsr;
175	/** The guest's TPR value used for TPR shadowing. */
176	uint8_t u8GuestTpr;
177	/** Alignment. */
178	uint8_t abAlignment0[6];
179
180	/** The basic VM-exit reason. */
181	uint16_t uExitReason;
182	/** Alignment. */
183	uint16_t u16Alignment0;
184	/** The VM-exit interruption error code. */
185	uint32_t uExitIntrErrorCode;
186	/** The VM-exit exit qualification. */
187	uint64_t uExitQualification;
188
189	/** The VM-exit interruption-information field. */
190	uint32_t uExitIntrInfo;
191	/** The VM-exit instruction-length field. */
192	uint32_t cbInstr;
193	/** Whether the VM-entry failed or not. */
194	bool fVMEntryFailed;
195	/** Alignment. */
196	uint8_t abAlignment1[5];
197
198	/** The VM-entry interruption-information field. */
199	uint32_t uEntryIntrInfo;
200	/** The VM-entry exception error code field. */
201	uint32_t uEntryXcptErrorCode;
202	/** The VM-entry instruction length field. */
203	uint32_t cbEntryInstr;
204
205	/** IDT-vectoring information field. */
206	uint32_t uIdtVectoringInfo;
207	/** IDT-vectoring error code. */
208	uint32_t uIdtVectoringErrorCode;
209
210	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
211	uint32_t fVmcsFieldsRead;
212	/** Whether TSC-offsetting should be setup before VM-entry. */
213	bool fUpdateTscOffsettingAndPreemptTimer;
214	/** Whether the VM-exit was caused by a page-fault during delivery of a
215	* contributary exception or a page-fault. */
216	bool fVectoringPF;
217	} VMXTRANSIENT, *PVMXTRANSIENT;
218	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
219	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
220	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
221
222
223	/**
224	* MSR-bitmap read permissions.
225	*/
226	typedef enum VMXMSREXITREAD
227	{
228	/** Reading this MSR causes a VM-exit. */
229	VMXMSREXIT_INTERCEPT_READ = 0xb,
230	/** Reading this MSR does not cause a VM-exit. */
231	VMXMSREXIT_PASSTHRU_READ
232	} VMXMSREXITREAD;
233
234	/**
235	* MSR-bitmap write permissions.
236	*/
237	typedef enum VMXMSREXITWRITE
238	{
239	/** Writing to this MSR causes a VM-exit. */
240	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
241	/** Writing to this MSR does not cause a VM-exit. */
242	VMXMSREXIT_PASSTHRU_WRITE
243	} VMXMSREXITWRITE;
244
245
246	/*******************************************************************************
247	* Internal Functions *
248	*******************************************************************************/
249	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
250	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
251	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
252	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
253	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
254	#endif
255	#ifndef HMVMX_USE_FUNCTION_TABLE
256	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
257	#define HMVMX_EXIT_DECL static int
258	#else
259	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
260	#endif
261
262	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306
307	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
309	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314
315
316	/*******************************************************************************
317	* Global Variables *
318	*******************************************************************************/
319	#ifdef HMVMX_USE_FUNCTION_TABLE
320	/**
321	* VM-exit handler.
322	*
323	* @returns VBox status code.
324	* @param pVCpu Pointer to the VMCPU.
325	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
326	* out-of-sync. Make sure to update the required
327	* fields before using them.
328	* @param pVmxTransient Pointer to the VMX-transient structure.
329	*/
330	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331	/** Pointer to VM-exit handler. */
332	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
333
334	/**
335	* VMX_EXIT dispatch table.
336	*/
337	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
338	{
339	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
340	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
341	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
342	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
343	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
344	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
345	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
346	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
347	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
348	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
349	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
350	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
351	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
352	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
353	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
354	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
355	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
356	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
357	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
358	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
359	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
360	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
361	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
362	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
363	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
364	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
365	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
366	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
367	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
368	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
369	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
370	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
371	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
372	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
373	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
374	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
375	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
376	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
377	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
378	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
379	/* 40 UNDEFINED */ hmR0VmxExitPause,
380	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
381	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
382	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
383	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
384	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
385	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
386	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
387	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
388	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
389	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
390	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
391	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
392	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
393	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
394	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
395	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
396	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
397	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
398	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
399	};
400	#endif /* HMVMX_USE_FUNCTION_TABLE */
401
402	#ifdef VBOX_STRICT
403	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
404	{
405	/* 0 */ "(Not Used)",
406	/* 1 */ "VMCALL executed in VMX root operation.",
407	/* 2 */ "VMCLEAR with invalid physical address.",
408	/* 3 */ "VMCLEAR with VMXON pointer.",
409	/* 4 */ "VMLAUNCH with non-clear VMCS.",
410	/* 5 */ "VMRESUME with non-launched VMCS.",
411	/* 6 */ "VMRESUME after VMXOFF",
412	/* 7 */ "VM entry with invalid control fields.",
413	/* 8 */ "VM entry with invalid host state fields.",
414	/* 9 */ "VMPTRLD with invalid physical address.",
415	/* 10 */ "VMPTRLD with VMXON pointer.",
416	/* 11 */ "VMPTRLD with incorrect revision identifier.",
417	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
418	/* 13 */ "VMWRITE to read-only VMCS component.",
419	/* 14 */ "(Not Used)",
420	/* 15 */ "VMXON executed in VMX root operation.",
421	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
422	/* 17 */ "VM entry with non-launched executing VMCS.",
423	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
424	/* 19 */ "VMCALL with non-clear VMCS.",
425	/* 20 */ "VMCALL with invalid VM-exit control fields.",
426	/* 21 */ "(Not Used)",
427	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
428	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
429	/* 24 */ "VMCALL with invalid SMM-monitor features.",
430	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
431	/* 26 */ "VM entry with events blocked by MOV SS.",
432	/* 27 */ "(Not Used)",
433	/* 28 */ "Invalid operand to INVEPT/INVVPID."
434	};
435	#endif /* VBOX_STRICT */
436
437
438
439	/**
440	* Updates the VM's last error record. If there was a VMX instruction error,
441	* reads the error data from the VMCS and updates VCPU's last error record as
442	* well.
443	*
444	* @param pVM Pointer to the VM.
445	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
446	* VERR_VMX_UNABLE_TO_START_VM or
447	* VERR_VMX_INVALID_VMCS_FIELD).
448	* @param rc The error code.
449	*/
450	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
451	{
452	AssertPtr(pVM);
453	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
454	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
455	{
456	AssertPtrReturnVoid(pVCpu);
457	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
458	}
459	pVM->hm.s.lLastError = rc;
460	}
461
462
463	/**
464	* Reads the VM-entry interruption-information field from the VMCS into the VMX
465	* transient structure.
466	*
467	* @returns VBox status code.
468	* @param pVmxTransient Pointer to the VMX transient structure.
469	*
470	* @remarks No-long-jump zone!!!
471	*/
472	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
473	{
474	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
475	AssertRCReturn(rc, rc);
476	return VINF_SUCCESS;
477	}
478
479
480	/**
481	* Reads the VM-entry exception error code field from the VMCS into
482	* the VMX transient structure.
483	*
484	* @returns VBox status code.
485	* @param pVmxTransient Pointer to the VMX transient structure.
486	*
487	* @remarks No-long-jump zone!!!
488	*/
489	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
490	{
491	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
492	AssertRCReturn(rc, rc);
493	return VINF_SUCCESS;
494	}
495
496
497	/**
498	* Reads the VM-entry exception error code field from the VMCS into
499	* the VMX transient structure.
500	*
501	* @returns VBox status code.
502	* @param pVCpu Pointer to the VMCPU.
503	* @param pVmxTransient Pointer to the VMX transient structure.
504	*
505	* @remarks No-long-jump zone!!!
506	*/
507	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
508	{
509	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
510	AssertRCReturn(rc, rc);
511	return VINF_SUCCESS;
512	}
513
514
515	/**
516	* Reads the VM-exit interruption-information field from the VMCS into the VMX
517	* transient structure.
518	*
519	* @returns VBox status code.
520	* @param pVCpu Pointer to the VMCPU.
521	* @param pVmxTransient Pointer to the VMX transient structure.
522	*/
523	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
524	{
525	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
526	{
527	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
528	AssertRCReturn(rc, rc);
529	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
530	}
531	return VINF_SUCCESS;
532	}
533
534
535	/**
536	* Reads the VM-exit interruption error code from the VMCS into the VMX
537	* transient structure.
538	*
539	* @returns VBox status code.
540	* @param pVCpu Pointer to the VMCPU.
541	* @param pVmxTransient Pointer to the VMX transient structure.
542	*/
543	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
544	{
545	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
546	{
547	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
548	AssertRCReturn(rc, rc);
549	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
550	}
551	return VINF_SUCCESS;
552	}
553
554
555	/**
556	* Reads the VM-exit instruction length field from the VMCS into the VMX
557	* transient structure.
558	*
559	* @returns VBox status code.
560	* @param pVCpu Pointer to the VMCPU.
561	* @param pVmxTransient Pointer to the VMX transient structure.
562	*/
563	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
564	{
565	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
566	{
567	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
568	AssertRCReturn(rc, rc);
569	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
570	}
571	return VINF_SUCCESS;
572	}
573
574
575	/**
576	* Reads the exit qualification from the VMCS into the VMX transient structure.
577	*
578	* @returns VBox status code.
579	* @param pVCpu Pointer to the VMCPU.
580	* @param pVmxTransient Pointer to the VMX transient structure.
581	*/
582	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
583	{
584	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
585	{
586	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
587	AssertRCReturn(rc, rc);
588	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
589	}
590	return VINF_SUCCESS;
591	}
592
593
594	/**
595	* Reads the IDT-vectoring information field from the VMCS into the VMX
596	* transient structure.
597	*
598	* @returns VBox status code.
599	* @param pVmxTransient Pointer to the VMX transient structure.
600	*
601	* @remarks No-long-jump zone!!!
602	*/
603	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the IDT-vectoring error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Enters VMX root mode operation on the current CPU.
636	*
637	* @returns VBox status code.
638	* @param pVM Pointer to the VM (optional, can be NULL, after
639	* a resume).
640	* @param HCPhysCpuPage Physical address of the VMXON region.
641	* @param pvCpuPage Pointer to the VMXON region.
642	*/
643	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
644	{
645	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
646	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
647	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
648
649	if (pVM)
650	{
651	/* Write the VMCS revision dword to the VMXON region. */
652	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
653	}
654
655	/* Enable the VMX bit in CR4 if necessary. */
656	RTCCUINTREG uCr4 = ASMGetCR4();
657	if (!(uCr4 & X86_CR4_VMXE))
658	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
659
660	/* Enter VMX root mode. */
661	int rc = VMXEnable(HCPhysCpuPage);
662	if (RT_FAILURE(rc))
663	ASMSetCR4(uCr4);
664
665	return rc;
666	}
667
668
669	/**
670	* Exits VMX root mode operation on the current CPU.
671	*
672	* @returns VBox status code.
673	*/
674	static int hmR0VmxLeaveRootMode(void)
675	{
676	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
677
678	/* If we're for some reason not in VMX root mode, then don't leave it. */
679	if (ASMGetCR4() & X86_CR4_VMXE)
680	{
681	/* Exit VMX root mode and clear the VMX bit in CR4 */
682	VMXDisable();
683	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
684	return VINF_SUCCESS;
685	}
686
687	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
688	}
689
690
691	/**
692	* Allocates and maps one physically contiguous page. The allocated page is
693	* zero'd out. (Used by various VT-x structures).
694	*
695	* @returns IPRT status code.
696	* @param pMemObj Pointer to the ring-0 memory object.
697	* @param ppVirt Where to store the virtual address of the
698	* allocation.
699	* @param pPhys Where to store the physical address of the
700	* allocation.
701	*/
702	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
703	{
704	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
705	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
706	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
707
708	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
709	if (RT_FAILURE(rc))
710	return rc;
711	ppVirt = RTR0MemObjAddress(pMemObj);
712	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
713	ASMMemZero32(*ppVirt, PAGE_SIZE);
714	return VINF_SUCCESS;
715	}
716
717
718	/**
719	* Frees and unmaps an allocated physical page.
720	*
721	* @param pMemObj Pointer to the ring-0 memory object.
722	* @param ppVirt Where to re-initialize the virtual address of
723	* allocation as 0.
724	* @param pHCPhys Where to re-initialize the physical address of the
725	* allocation as 0.
726	*/
727	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
728	{
729	AssertPtr(pMemObj);
730	AssertPtr(ppVirt);
731	AssertPtr(pHCPhys);
732	if (*pMemObj != NIL_RTR0MEMOBJ)
733	{
734	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
735	AssertRC(rc);
736	*pMemObj = NIL_RTR0MEMOBJ;
737	*ppVirt = 0;
738	*pHCPhys = 0;
739	}
740	}
741
742
743	/**
744	* Worker function to free VT-x related structures.
745	*
746	* @returns IPRT status code.
747	* @param pVM Pointer to the VM.
748	*/
749	static void hmR0VmxStructsFree(PVM pVM)
750	{
751	for (VMCPUID i = 0; i < pVM->cCpus; i++)
752	{
753	PVMCPU pVCpu = &pVM->aCpus[i];
754	AssertPtr(pVCpu);
755
756	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
757	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
758	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
759	#endif
760
761	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
762	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
763
764	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
765	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
766	}
767
768	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
769	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
770	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
771	#endif
772	}
773
774
775	/**
776	* Worker function to allocate VT-x related VM structures.
777	*
778	* @returns IPRT status code.
779	* @param pVM Pointer to the VM.
780	*/
781	static int hmR0VmxStructsAlloc(PVM pVM)
782	{
783	/*
784	* Initialize members up-front so we can cleanup properly on allocation failure.
785	*/
786	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
787	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
788	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
789	pVM->hm.s.vmx.HCPhys##a_Name = 0;
790
791	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
792	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
793	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
794	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
795
796	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
797	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
798	#endif
799	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
800
801	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
802	for (VMCPUID i = 0; i < pVM->cCpus; i++)
803	{
804	PVMCPU pVCpu = &pVM->aCpus[i];
805	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
806	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
807	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
808	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
809	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
810	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
811	#endif
812	}
813	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
814	#undef VMXLOCAL_INIT_VM_MEMOBJ
815
816	/*
817	* Allocate all the VT-x structures.
818	*/
819	int rc = VINF_SUCCESS;
820	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
821	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
822	if (RT_FAILURE(rc))
823	goto cleanup;
824	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
825	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
826	#endif
827
828	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
829	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
830	{
831	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
832	&pVM->hm.s.vmx.HCPhysApicAccess);
833	if (RT_FAILURE(rc))
834	goto cleanup;
835	}
836
837	/*
838	* Initialize per-VCPU VT-x structures.
839	*/
840	for (VMCPUID i = 0; i < pVM->cCpus; i++)
841	{
842	PVMCPU pVCpu = &pVM->aCpus[i];
843	AssertPtr(pVCpu);
844
845	/* Allocate the VM control structure (VMCS). */
846	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
847	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
848	if (RT_FAILURE(rc))
849	goto cleanup;
850
851	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
852	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
853	{
854	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
855	&pVCpu->hm.s.vmx.HCPhysVirtApic);
856	if (RT_FAILURE(rc))
857	goto cleanup;
858	}
859
860	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
861	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
862	{
863	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
864	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
865	if (RT_FAILURE(rc))
866	goto cleanup;
867	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
868	}
869
870	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
871	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
872	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
873	if (RT_FAILURE(rc))
874	goto cleanup;
875
876	/* Allocate the VM-exit MSR-load page for the host MSRs. */
877	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
878	if (RT_FAILURE(rc))
879	goto cleanup;
880	#endif
881	}
882
883	return VINF_SUCCESS;
884
885	cleanup:
886	hmR0VmxStructsFree(pVM);
887	return rc;
888	}
889
890
891	/**
892	* Does global VT-x initialization (called during module initialization).
893	*
894	* @returns VBox status code.
895	*/
896	VMMR0DECL(int) VMXR0GlobalInit(void)
897	{
898	#ifdef HMVMX_USE_FUNCTION_TABLE
899	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
900	# ifdef VBOX_STRICT
901	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
902	Assert(g_apfnVMExitHandlers[i]);
903	# endif
904	#endif
905	return VINF_SUCCESS;
906	}
907
908
909	/**
910	* Does global VT-x termination (called during module termination).
911	*/
912	VMMR0DECL(void) VMXR0GlobalTerm()
913	{
914	/* Nothing to do currently. */
915	}
916
917
918	/**
919	* Sets up and activates VT-x on the current CPU.
920	*
921	* @returns VBox status code.
922	* @param pCpu Pointer to the global CPU info struct.
923	* @param pVM Pointer to the VM (can be NULL after a host resume
924	* operation).
925	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
926	* fEnabledByHost is true).
927	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
928	* @a fEnabledByHost is true).
929	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
930	* enable VT-x/AMD-V on the host.
931	*/
932	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
933	{
934	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
935	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
936
937	if (!fEnabledByHost)
938	{
939	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
940	if (RT_FAILURE(rc))
941	return rc;
942	}
943
944	/*
945	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
946	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
947	* each time while reusing a VPID after hitting the MaxASID limit once.
948	*/
949	if ( pVM
950	&& pVM->hm.s.vmx.fVpid
951	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
952	{
953	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
954	pCpu->fFlushAsidBeforeUse = false;
955	}
956	else
957	pCpu->fFlushAsidBeforeUse = true;
958
959	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
960	++pCpu->cTlbFlushes;
961
962	return VINF_SUCCESS;
963	}
964
965
966	/**
967	* Deactivates VT-x on the current CPU.
968	*
969	* @returns VBox status code.
970	* @param pCpu Pointer to the global CPU info struct.
971	* @param pvCpuPage Pointer to the VMXON region.
972	* @param HCPhysCpuPage Physical address of the VMXON region.
973	*/
974	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
975	{
976	NOREF(pCpu);
977	NOREF(pvCpuPage);
978	NOREF(HCPhysCpuPage);
979
980	return hmR0VmxLeaveRootMode();
981	}
982
983
984	/**
985	* Sets the permission bits for the specified MSR in the MSR bitmap.
986	*
987	* @param pVCpu Pointer to the VMCPU.
988	* @param uMSR The MSR value.
989	* @param enmRead Whether reading this MSR causes a VM-exit.
990	* @param enmWrite Whether writing this MSR causes a VM-exit.
991	*/
992	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
993	{
994	int32_t iBit;
995	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
996
997	/*
998	* Layout:
999	* 0x000 - 0x3ff - Low MSR read bits
1000	* 0x400 - 0x7ff - High MSR read bits
1001	* 0x800 - 0xbff - Low MSR write bits
1002	* 0xc00 - 0xfff - High MSR write bits
1003	*/
1004	if (uMsr <= 0x00001FFF)
1005	iBit = uMsr;
1006	else if ( uMsr >= 0xC0000000
1007	&& uMsr <= 0xC0001FFF)
1008	{
1009	iBit = (uMsr - 0xC0000000);
1010	pbMsrBitmap += 0x400;
1011	}
1012	else
1013	{
1014	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1015	return;
1016	}
1017
1018	Assert(iBit <= 0x1fff);
1019	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1020	ASMBitSet(pbMsrBitmap, iBit);
1021	else
1022	ASMBitClear(pbMsrBitmap, iBit);
1023
1024	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1025	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1026	else
1027	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1028	}
1029
1030
1031	/**
1032	* Flushes the TLB using EPT.
1033	*
1034	* @returns VBox status code.
1035	* @param pVM Pointer to the VM.
1036	* @param pVCpu Pointer to the VMCPU.
1037	* @param enmFlush Type of flush.
1038	*/
1039	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1040	{
1041	AssertPtr(pVM);
1042	Assert(pVM->hm.s.fNestedPaging);
1043
1044	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1045
1046	uint64_t descriptor[2];
1047	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1048	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1049
1050	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1051	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1052	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1053	}
1054
1055
1056	/**
1057	* Flushes the TLB using VPID.
1058	*
1059	* @returns VBox status code.
1060	* @param pVM Pointer to the VM.
1061	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1062	* enmFlush).
1063	* @param enmFlush Type of flush.
1064	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1065	* on @a enmFlush).
1066	*/
1067	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1068	{
1069	AssertPtr(pVM);
1070	Assert(pVM->hm.s.vmx.fVpid);
1071
1072	uint64_t descriptor[2];
1073	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1074	{
1075	descriptor[0] = 0;
1076	descriptor[1] = 0;
1077	}
1078	else
1079	{
1080	AssertPtr(pVCpu);
1081	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1082	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1083	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1084	descriptor[1] = GCPtr;
1085	}
1086
1087	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1088	AssertMsg(rc == VINF_SUCCESS,
1089	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1090	if ( RT_SUCCESS(rc)
1091	&& pVCpu)
1092	{
1093	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1094	}
1095	}
1096
1097
1098	/**
1099	* Invalidates a guest page by guest virtual address. Only relevant for
1100	* EPT/VPID, otherwise there is nothing really to invalidate.
1101	*
1102	* @returns VBox status code.
1103	* @param pVM Pointer to the VM.
1104	* @param pVCpu Pointer to the VMCPU.
1105	* @param GCVirt Guest virtual address of the page to invalidate.
1106	*/
1107	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1108	{
1109	AssertPtr(pVM);
1110	AssertPtr(pVCpu);
1111	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1112
1113	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1114	if (!fFlushPending)
1115	{
1116	/*
1117	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1118	* See @bugref{6043} and @bugref{6177}.
1119	*
1120	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1121	* function maybe called in a loop with individual addresses.
1122	*/
1123	if (pVM->hm.s.vmx.fVpid)
1124	{
1125	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1126	{
1127	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1128	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1129	}
1130	else
1131	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1132	}
1133	else if (pVM->hm.s.fNestedPaging)
1134	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1135	}
1136
1137	return VINF_SUCCESS;
1138	}
1139
1140
1141	/**
1142	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1143	* otherwise there is nothing really to invalidate.
1144	*
1145	* @returns VBox status code.
1146	* @param pVM Pointer to the VM.
1147	* @param pVCpu Pointer to the VMCPU.
1148	* @param GCPhys Guest physical address of the page to invalidate.
1149	*/
1150	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1151	{
1152	LogFlowFunc(("%RGp\n", GCPhys));
1153
1154	/*
1155	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1156	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1157	* This function might be called in a loop.
1158	*/
1159	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1160	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1161	return VINF_SUCCESS;
1162	}
1163
1164
1165	/**
1166	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1167	* case where neither EPT nor VPID is supported by the CPU.
1168	*
1169	* @param pVM Pointer to the VM.
1170	* @param pVCpu Pointer to the VMCPU.
1171	*
1172	* @remarks Called with interrupts disabled.
1173	*/
1174	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1175	{
1176	NOREF(pVM);
1177	AssertPtr(pVCpu);
1178	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1179	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1180
1181	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1182	AssertPtr(pCpu);
1183
1184	pVCpu->hm.s.TlbShootdown.cPages = 0;
1185	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1186	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1187	pVCpu->hm.s.fForceTLBFlush = false;
1188	return;
1189	}
1190
1191
1192	/**
1193	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1194	*
1195	* @param pVM Pointer to the VM.
1196	* @param pVCpu Pointer to the VMCPU.
1197	* @remarks All references to "ASID" in this function pertains to "VPID" in
1198	* Intel's nomenclature. The reason is, to avoid confusion in compare
1199	* statements since the host-CPU copies are named "ASID".
1200	*
1201	* @remarks Called with interrupts disabled.
1202	*/
1203	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1204	{
1205	AssertPtr(pVM);
1206	AssertPtr(pVCpu);
1207	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1208	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1209	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1210
1211	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1212	AssertPtr(pCpu);
1213
1214	/*
1215	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1216	* This can happen both for start & resume due to long jumps back to ring-3.
1217	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1218	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1219	*/
1220	bool fNewASID = false;
1221	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1222	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1223	{
1224	pVCpu->hm.s.fForceTLBFlush = true;
1225	fNewASID = true;
1226	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1227	}
1228
1229	/*
1230	* Check for explicit TLB shootdowns.
1231	*/
1232	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1233	{
1234	pVCpu->hm.s.fForceTLBFlush = true;
1235	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1236	}
1237
1238	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1239	if (pVCpu->hm.s.fForceTLBFlush)
1240	{
1241	if (fNewASID)
1242	{
1243	++pCpu->uCurrentAsid;
1244	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1245	{
1246	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1247	pCpu->cTlbFlushes++;
1248	pCpu->fFlushAsidBeforeUse = true;
1249	}
1250
1251	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1252	if (pCpu->fFlushAsidBeforeUse)
1253	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1254	}
1255	else
1256	{
1257	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1258	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1259	else
1260	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1261	}
1262
1263	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1264	pVCpu->hm.s.fForceTLBFlush = false;
1265	}
1266	else
1267	{
1268	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1269	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1270	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1271	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1272
1273	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1274	* not be executed. See hmQueueInvlPage() where it is commented
1275	* out. Support individual entry flushing someday. */
1276	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1277	{
1278	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1279
1280	/*
1281	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1282	* as supported by the CPU.
1283	*/
1284	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1285	{
1286	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1287	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1288	}
1289	else
1290	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1291	}
1292	else
1293	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1294	}
1295	pVCpu->hm.s.TlbShootdown.cPages = 0;
1296	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1297
1298	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1299	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1300	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1301	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1302	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1303	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1304
1305	/* Update VMCS with the VPID. */
1306	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1307	AssertRC(rc);
1308	}
1309
1310
1311	/**
1312	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1313	*
1314	* @returns VBox status code.
1315	* @param pVM Pointer to the VM.
1316	* @param pVCpu Pointer to the VMCPU.
1317	*
1318	* @remarks Called with interrupts disabled.
1319	*/
1320	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1321	{
1322	AssertPtr(pVM);
1323	AssertPtr(pVCpu);
1324	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1325	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1326
1327	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1328	AssertPtr(pCpu);
1329
1330	/*
1331	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1332	* This can happen both for start & resume due to long jumps back to ring-3.
1333	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1334	*/
1335	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1336	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1337	{
1338	pVCpu->hm.s.fForceTLBFlush = true;
1339	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1340	}
1341
1342	/* Check for explicit TLB shootdown flushes. */
1343	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1344	{
1345	pVCpu->hm.s.fForceTLBFlush = true;
1346	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1347	}
1348
1349	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1350	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1351
1352	if (pVCpu->hm.s.fForceTLBFlush)
1353	{
1354	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1355	pVCpu->hm.s.fForceTLBFlush = false;
1356	}
1357	else
1358	{
1359	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1360	* not be executed. See hmQueueInvlPage() where it is commented
1361	* out. Support individual entry flushing someday. */
1362	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1363	{
1364	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1365	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1366	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1367	}
1368	else
1369	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1370	}
1371
1372	pVCpu->hm.s.TlbShootdown.cPages = 0;
1373	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1374	}
1375
1376
1377	/**
1378	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1379	*
1380	* @returns VBox status code.
1381	* @param pVM Pointer to the VM.
1382	* @param pVCpu Pointer to the VMCPU.
1383	*
1384	* @remarks Called with interrupts disabled.
1385	*/
1386	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1387	{
1388	AssertPtr(pVM);
1389	AssertPtr(pVCpu);
1390	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1391	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1392
1393	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1394
1395	/*
1396	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1397	* This can happen both for start & resume due to long jumps back to ring-3.
1398	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1399	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1400	*/
1401	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1402	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1403	{
1404	pVCpu->hm.s.fForceTLBFlush = true;
1405	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1406	}
1407
1408	/* Check for explicit TLB shootdown flushes. */
1409	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1410	{
1411	/*
1412	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1413	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1414	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1415	*/
1416	pVCpu->hm.s.fForceTLBFlush = true;
1417	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1418	}
1419
1420	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1421	if (pVCpu->hm.s.fForceTLBFlush)
1422	{
1423	++pCpu->uCurrentAsid;
1424	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1425	{
1426	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1427	pCpu->fFlushAsidBeforeUse = true;
1428	pCpu->cTlbFlushes++;
1429	}
1430
1431	pVCpu->hm.s.fForceTLBFlush = false;
1432	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1433	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1434	if (pCpu->fFlushAsidBeforeUse)
1435	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1436	}
1437	else
1438	{
1439	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1440	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1441	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1442	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1443
1444	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1445	* not be executed. See hmQueueInvlPage() where it is commented
1446	* out. Support individual entry flushing someday. */
1447	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1448	{
1449	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1450	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1451	{
1452	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1453	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1454	}
1455	else
1456	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1457	}
1458	else
1459	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1460	}
1461
1462	pVCpu->hm.s.TlbShootdown.cPages = 0;
1463	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1464
1465	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1466	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1467	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1468	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1469	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1470	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1471
1472	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1473	AssertRC(rc);
1474	}
1475
1476
1477	/**
1478	* Flushes the guest TLB entry based on CPU capabilities.
1479	*
1480	* @param pVCpu Pointer to the VMCPU.
1481	*/
1482	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1483	{
1484	PVM pVM = pVCpu->CTX_SUFF(pVM);
1485	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1486	{
1487	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1488	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1489	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1490	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1491	default:
1492	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1493	break;
1494	}
1495	}
1496
1497
1498	/**
1499	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1500	* TLB entries from the host TLB before VM-entry.
1501	*
1502	* @returns VBox status code.
1503	* @param pVM Pointer to the VM.
1504	*/
1505	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1506	{
1507	/*
1508	* Determine optimal flush type for nested paging.
1509	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1510	* guest execution (see hmR3InitFinalizeR0()).
1511	*/
1512	if (pVM->hm.s.fNestedPaging)
1513	{
1514	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1515	{
1516	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1517	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1518	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1519	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1520	else
1521	{
1522	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1523	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1524	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1525	}
1526
1527	/* Make sure the write-back cacheable memory type for EPT is supported. */
1528	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1529	{
1530	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1531	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1532	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1533	}
1534	}
1535	else
1536	{
1537	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1538	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1539	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1540	}
1541	}
1542
1543	/*
1544	* Determine optimal flush type for VPID.
1545	*/
1546	if (pVM->hm.s.vmx.fVpid)
1547	{
1548	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1549	{
1550	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1551	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1552	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1553	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1554	else
1555	{
1556	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1557	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1558	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1559	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1560	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1561	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1562	pVM->hm.s.vmx.fVpid = false;
1563	}
1564	}
1565	else
1566	{
1567	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1568	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1569	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1570	pVM->hm.s.vmx.fVpid = false;
1571	}
1572	}
1573
1574	/*
1575	* Setup the handler for flushing tagged-TLBs.
1576	*/
1577	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1578	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1579	else if (pVM->hm.s.fNestedPaging)
1580	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1581	else if (pVM->hm.s.vmx.fVpid)
1582	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1583	else
1584	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1585	return VINF_SUCCESS;
1586	}
1587
1588
1589	/**
1590	* Sets up pin-based VM-execution controls in the VMCS.
1591	*
1592	* @returns VBox status code.
1593	* @param pVM Pointer to the VM.
1594	* @param pVCpu Pointer to the VMCPU.
1595	*/
1596	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1597	{
1598	AssertPtr(pVM);
1599	AssertPtr(pVCpu);
1600
1601	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1602	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1603
1604	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1605	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1606	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1607
1608	/* Enable the VMX preemption timer. */
1609	if (pVM->hm.s.vmx.fUsePreemptTimer)
1610	{
1611	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1612	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1613	}
1614
1615	if ((val & zap) != val)
1616	{
1617	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1618	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1619	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1620	}
1621
1622	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1623	AssertRCReturn(rc, rc);
1624
1625	/* Update VCPU with the currently set pin-based VM-execution controls. */
1626	pVCpu->hm.s.vmx.u32PinCtls = val;
1627	return rc;
1628	}
1629
1630
1631	/**
1632	* Sets up processor-based VM-execution controls in the VMCS.
1633	*
1634	* @returns VBox status code.
1635	* @param pVM Pointer to the VM.
1636	* @param pVMCPU Pointer to the VMCPU.
1637	*/
1638	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1639	{
1640	AssertPtr(pVM);
1641	AssertPtr(pVCpu);
1642
1643	int rc = VERR_INTERNAL_ERROR_5;
1644	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1645	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1646
1647	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1648	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1649	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1650	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1651	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1652	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1653	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1654
1655	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1656	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1657	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1658	{
1659	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1660	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1661	}
1662
1663	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1664	if (!pVM->hm.s.fNestedPaging)
1665	{
1666	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1667	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1668	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1669	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1670	}
1671
1672	/* Use TPR shadowing if supported by the CPU. */
1673	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1674	{
1675	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1676	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1677	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1678	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1679	AssertRCReturn(rc, rc);
1680
1681	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1682	/* CR8 writes causes a VM-exit based on TPR threshold. */
1683	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1684	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1685	}
1686	else
1687	{
1688	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1689	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1690	}
1691
1692	/* Use MSR-bitmaps if supported by the CPU. */
1693	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1694	{
1695	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1696
1697	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1698	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1699	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1700	AssertRCReturn(rc, rc);
1701
1702	/*
1703	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1704	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1705	*/
1706	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1707	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1708	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1709	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1710	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1711	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1712	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1713	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1714	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1715	}
1716
1717	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1718	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1719	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1720
1721	if ((val & zap) != val)
1722	{
1723	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1724	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1725	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1726	}
1727
1728	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1729	AssertRCReturn(rc, rc);
1730
1731	/* Update VCPU with the currently set processor-based VM-execution controls. */
1732	pVCpu->hm.s.vmx.u32ProcCtls = val;
1733
1734	/*
1735	* Secondary processor-based VM-execution controls.
1736	*/
1737	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1738	{
1739	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1740	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1741
1742	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1743	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1744
1745	if (pVM->hm.s.fNestedPaging)
1746	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1747	else
1748	{
1749	/*
1750	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1751	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1752	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1753	*/
1754	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1755	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1756	}
1757
1758	if (pVM->hm.s.vmx.fVpid)
1759	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1760
1761	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1762	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1763
1764	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1765	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1766	* done dynamically. */
1767	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1768	{
1769	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1770	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1771	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1772	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1773	AssertRCReturn(rc, rc);
1774	}
1775
1776	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1777	{
1778	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1779	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1780	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1781	}
1782
1783	if ((val & zap) != val)
1784	{
1785	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1786	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1787	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1788	}
1789
1790	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1791	AssertRCReturn(rc, rc);
1792
1793	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1794	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1795	}
1796
1797	return VINF_SUCCESS;
1798	}
1799
1800
1801	/**
1802	* Sets up miscellaneous (everything other than Pin & Processor-based
1803	* VM-execution) control fields in the VMCS.
1804	*
1805	* @returns VBox status code.
1806	* @param pVM Pointer to the VM.
1807	* @param pVCpu Pointer to the VMCPU.
1808	*/
1809	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1810	{
1811	AssertPtr(pVM);
1812	AssertPtr(pVCpu);
1813
1814	/** @todo Shouldn't we able to avoid initializing with 0? */
1815	int rc = VERR_GENERAL_FAILURE;
1816
1817	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1818	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1819	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1820
1821	/*
1822	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1823	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1824	* We thus use the exception bitmap to control it rather than use both.
1825	*/
1826	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1827	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1828
1829	/** @todo Explore possibility of using IO-bitmaps. */
1830	/* All IO & IOIO instructions cause VM-exits. */
1831	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1832	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1833
1834	/* Initialize the MSR-bitmap area. */
1835	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1836	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1837	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1838
1839	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1840	/* Setup MSR autoloading/storing. */
1841	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1842	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1843	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1844	AssertRCReturn(rc, rc);
1845	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1846	AssertRCReturn(rc, rc);
1847
1848	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1849	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1850	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1851	AssertRCReturn(rc, rc);
1852	#endif
1853
1854	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1855	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1856	AssertRCReturn(rc, rc);
1857
1858	/* Setup debug controls */
1859	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1860	AssertRCReturn(rc, rc);
1861	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1862	AssertRCReturn(rc, rc);
1863	return rc;
1864	}
1865
1866
1867	/**
1868	* Sets up the initial exception bitmap in the VMCS based on static conditions
1869	* (i.e. conditions that cannot ever change at runtime).
1870	*
1871	* @returns VBox status code.
1872	* @param pVM Pointer to the VM.
1873	* @param pVCpu Pointer to the VMCPU.
1874	*/
1875	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1876	{
1877	AssertPtr(pVM);
1878	AssertPtr(pVCpu);
1879
1880	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1881
1882	uint32_t u32XcptBitmap = 0;
1883
1884	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1885	if (!pVM->hm.s.fNestedPaging)
1886	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1887
1888	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1889	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1890	AssertRCReturn(rc, rc);
1891	return rc;
1892	}
1893
1894
1895	/**
1896	* Sets up the initial guest-state mask. The guest-state mask is consulted
1897	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1898	* for the nested virtualization case (as it would cause a VM-exit).
1899	*
1900	* @param pVCpu Pointer to the VMCPU.
1901	*/
1902	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1903	{
1904	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1905	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1906	return VINF_SUCCESS;
1907	}
1908
1909
1910	/**
1911	* Does per-VM VT-x initialization.
1912	*
1913	* @returns VBox status code.
1914	* @param pVM Pointer to the VM.
1915	*/
1916	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1917	{
1918	LogFlowFunc(("pVM=%p\n", pVM));
1919
1920	int rc = hmR0VmxStructsAlloc(pVM);
1921	if (RT_FAILURE(rc))
1922	{
1923	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1924	return rc;
1925	}
1926
1927	return VINF_SUCCESS;
1928	}
1929
1930
1931	/**
1932	* Does per-VM VT-x termination.
1933	*
1934	* @returns VBox status code.
1935	* @param pVM Pointer to the VM.
1936	*/
1937	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1938	{
1939	LogFlowFunc(("pVM=%p\n", pVM));
1940
1941	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1942	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1943	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1944	#endif
1945	hmR0VmxStructsFree(pVM);
1946	return VINF_SUCCESS;
1947	}
1948
1949
1950	/**
1951	* Sets up the VM for execution under VT-x.
1952	* This function is only called once per-VM during initalization.
1953	*
1954	* @returns VBox status code.
1955	* @param pVM Pointer to the VM.
1956	*/
1957	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1958	{
1959	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1960	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1961
1962	LogFlowFunc(("pVM=%p\n", pVM));
1963
1964	/*
1965	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1966	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1967	*/
1968	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
1969	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1970	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1971	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1972	{
1973	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1974	return VERR_INTERNAL_ERROR;
1975	}
1976
1977	/* Initialize these always, see hmR3InitFinalizeR0().*/
1978	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1979	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1980
1981	/* Setup the tagged-TLB flush handlers. */
1982	int rc = hmR0VmxSetupTaggedTlb(pVM);
1983	if (RT_FAILURE(rc))
1984	{
1985	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1986	return rc;
1987	}
1988
1989	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1990	{
1991	PVMCPU pVCpu = &pVM->aCpus[i];
1992	AssertPtr(pVCpu);
1993	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1994
1995	/* Set revision dword at the beginning of the VMCS structure. */
1996	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1997
1998	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1999	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2000	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2001	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2002
2003	/* Load this VMCS as the current VMCS. */
2004	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2005	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2006	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2007
2008	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2009	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2010	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2011
2012	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2013	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2014	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2015
2016	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2017	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2018	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2019
2020	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2021	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2022	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2023
2024	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2025	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2026	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2027
2028	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2029	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2030	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2031	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2032	#endif
2033
2034	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2035	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2036	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2037	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2038
2039	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2040	}
2041
2042	return VINF_SUCCESS;
2043	}
2044
2045
2046	/**
2047	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2048	* the VMCS.
2049	*
2050	* @returns VBox status code.
2051	* @param pVM Pointer to the VM.
2052	* @param pVCpu Pointer to the VMCPU.
2053	*/
2054	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2055	{
2056	RTCCUINTREG uReg = ASMGetCR0();
2057	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2058	AssertRCReturn(rc, rc);
2059
2060	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2061	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2062	if (HMVMX_IS_64BIT_HOST_MODE())
2063	{
2064	uint64_t uRegCR3 = HMR0Get64bitCR3();
2065	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2066	}
2067	else
2068	#endif
2069	{
2070	uReg = ASMGetCR3();
2071	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2072	}
2073	AssertRCReturn(rc, rc);
2074
2075	uReg = ASMGetCR4();
2076	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2077	AssertRCReturn(rc, rc);
2078	return rc;
2079	}
2080
2081
2082	/**
2083	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2084	* the host-state area in the VMCS.
2085	*
2086	* @returns VBox status code.
2087	* @param pVM Pointer to the VM.
2088	* @param pVCpu Pointer to the VMCPU.
2089	*/
2090	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2091	{
2092	int rc = VERR_INTERNAL_ERROR_5;
2093	RTSEL uSelDS = 0;
2094	RTSEL uSelES = 0;
2095	RTSEL uSelFS = 0;
2096	RTSEL uSelGS = 0;
2097	RTSEL uSelTR = 0;
2098
2099	/*
2100	* Host DS, ES, FS and GS segment registers.
2101	*/
2102	#if HC_ARCH_BITS == 64
2103	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2104	uSelDS = ASMGetDS();
2105	uSelES = ASMGetES();
2106	uSelFS = ASMGetFS();
2107	uSelGS = ASMGetGS();
2108	#endif
2109
2110	/*
2111	* Host CS and SS segment registers.
2112	*/
2113	RTSEL uSelCS;
2114	RTSEL uSelSS;
2115	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2116	if (HMVMX_IS_64BIT_HOST_MODE())
2117	{
2118	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2119	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2120	}
2121	else
2122	{
2123	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2124	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2125	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2126	}
2127	#else
2128	uSelCS = ASMGetCS();
2129	uSelSS = ASMGetSS();
2130	#endif
2131
2132	/*
2133	* Host TR segment register.
2134	*/
2135	uSelTR = ASMGetTR();
2136
2137	#if HC_ARCH_BITS == 64
2138	/*
2139	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2140	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2141	*/
2142	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2143	{
2144	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2145	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2146	uSelDS = 0;
2147	}
2148	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2149	{
2150	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2151	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2152	uSelES = 0;
2153	}
2154	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2155	{
2156	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2157	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2158	uSelFS = 0;
2159	}
2160	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2161	{
2162	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2163	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2164	uSelGS = 0;
2165	}
2166	#endif
2167
2168	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2169	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2170	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2171	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2172	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2173	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2174	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2175	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2176	Assert(uSelCS);
2177	Assert(uSelTR);
2178
2179	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2180	#if 0
2181	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2182	Assert(uSelSS != 0);
2183	#endif
2184
2185	/* Write these host selector fields into the host-state area in the VMCS. */
2186	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2187	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2188	#if HC_ARCH_BITS == 64
2189	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2190	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2191	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2192	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2193	#endif
2194	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2195
2196	/*
2197	* Host GDTR and IDTR.
2198	*/
2199	RTGDTR Gdtr;
2200	RT_ZERO(Gdtr);
2201	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2202	if (HMVMX_IS_64BIT_HOST_MODE())
2203	{
2204	X86XDTR64 Gdtr64;
2205	X86XDTR64 Idtr64;
2206	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2207	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2208	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2209
2210	Gdtr.cbGdt = Gdtr64.cb;
2211	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2212	}
2213	else
2214	#endif
2215	{
2216	RTIDTR Idtr;
2217	ASMGetGDTR(&Gdtr);
2218	ASMGetIDTR(&Idtr);
2219	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2220	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2221
2222	#if HC_ARCH_BITS == 64
2223	/*
2224	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2225	* maximum limit (0xffff) on every VM-exit.
2226	*/
2227	if (Gdtr.cbGdt != 0xffff)
2228	{
2229	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2230	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2231	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2232	}
2233
2234	/*
2235	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2236	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2237	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2238	*/
2239	if (Idtr.cbIdt < 0x0fff)
2240	{
2241	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2242	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2243	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2244	}
2245	#endif
2246	}
2247
2248	/*
2249	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2250	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2251	*/
2252	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2253	{
2254	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2255	return VERR_VMX_INVALID_HOST_STATE;
2256	}
2257
2258	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2259	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2260	if (HMVMX_IS_64BIT_HOST_MODE())
2261	{
2262	/* We need the 64-bit TR base for hybrid darwin. */
2263	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2264	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2265	}
2266	else
2267	#endif
2268	{
2269	uintptr_t uTRBase;
2270	#if HC_ARCH_BITS == 64
2271	uTRBase = X86DESC64_BASE(pDesc);
2272	#else
2273	uTRBase = X86DESC_BASE(pDesc);
2274	#endif
2275	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2276	}
2277	AssertRCReturn(rc, rc);
2278
2279	/*
2280	* Host FS base and GS base.
2281	*/
2282	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2283	if (HMVMX_IS_64BIT_HOST_MODE())
2284	{
2285	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2286	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2287	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2288	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2289
2290	# if HC_ARCH_BITS == 64
2291	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2292	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2293	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2294	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2295	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2296	# endif
2297	}
2298	#endif
2299	return rc;
2300	}
2301
2302
2303	/**
2304	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2305	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2306	* the host after every successful VM exit.
2307	*
2308	* @returns VBox status code.
2309	* @param pVM Pointer to the VM.
2310	* @param pVCpu Pointer to the VMCPU.
2311	*/
2312	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2313	{
2314	AssertPtr(pVCpu);
2315	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2316
2317	int rc = VINF_SUCCESS;
2318	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2319	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2320	uint32_t cHostMsrs = 0;
2321	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2322
2323	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2324	{
2325	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2326
2327	#if HC_ARCH_BITS == 64
2328	/* Paranoia. 64-bit code requires these bits to be set always. */
2329	Assert(u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2330	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2331	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2332
2333	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2334	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2335	if (CPUMIsGuestInLongMode(pVCpu))
2336	{
2337	uint64_t u64GuestEfer;
2338	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2339	AssertRC(rc);
2340
2341	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2342	{
2343	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2344	pHostMsr->u32Reserved = 0;
2345	pHostMsr->u64Value = u64HostEfer;
2346	pHostMsr++; cHostMsrs++;
2347	}
2348	}
2349	#else /* HC_ARCH_BITS != 64 */
2350	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2351	pHostMsr->u32Reserved = 0;
2352	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2353	if (CPUMIsGuestInLongMode(pVCpu))
2354	{
2355	/* Must match the EFER value in our 64 bits switcher. */
2356	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2357	}
2358	else
2359	# endif
2360	pHostMsr->u64Value = u64HostEfer;
2361	pHostMsr++; cHostMsrs++;
2362	#endif /* HC_ARCH_BITS == 64 */
2363	}
2364
2365	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2366	if (HMVMX_IS_64BIT_HOST_MODE())
2367	{
2368	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2369	pHostMsr->u32Reserved = 0;
2370	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2371	pHostMsr++; cHostMsrs++;
2372	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2373	pHostMsr->u32Reserved = 0;
2374	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2375	pHostMsr++; cHostMsrs++;
2376	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2377	pHostMsr->u32Reserved = 0;
2378	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2379	pHostMsr++; cHostMsrs++;
2380	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2381	pHostMsr->u32Reserved = 0;
2382	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2383	pHostMsr++; cHostMsrs++;
2384	}
2385	# endif
2386
2387	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2388	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2389	{
2390	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2391	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2392	}
2393
2394	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2395	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2396
2397	/*
2398	* Host Sysenter MSRs.
2399	*/
2400	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2401	AssertRCReturn(rc, rc);
2402	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2403	if (HMVMX_IS_64BIT_HOST_MODE())
2404	{
2405	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2406	AssertRCReturn(rc, rc);
2407	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2408	}
2409	else
2410	{
2411	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2412	AssertRCReturn(rc, rc);
2413	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2414	}
2415	# elif HC_ARCH_BITS == 32
2416	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2417	AssertRCReturn(rc, rc);
2418	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2419	# else
2420	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2421	AssertRCReturn(rc, rc);
2422	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2423	# endif
2424	AssertRCReturn(rc, rc);
2425
2426	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2427	* hmR0VmxSetupExitCtls() !! */
2428	return rc;
2429	}
2430
2431
2432	/**
2433	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2434	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2435	* controls".
2436	*
2437	* @returns VBox status code.
2438	* @param pVCpu Pointer to the VMCPU.
2439	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2440	* out-of-sync. Make sure to update the required fields
2441	* before using them.
2442	*
2443	* @remarks No-long-jump zone!!!
2444	*/
2445	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2446	{
2447	int rc = VINF_SUCCESS;
2448	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2449	{
2450	PVM pVM = pVCpu->CTX_SUFF(pVM);
2451	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2452	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2453
2454	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2455	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2456
2457	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2458	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2459	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2460	else
2461	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2462
2463	/*
2464	* The following should not be set (since we're not in SMM mode):
2465	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2466	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2467	*/
2468
2469	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2470	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2471	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2472
2473	if ((val & zap) != val)
2474	{
2475	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2476	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2477	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2478	}
2479
2480	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2481	AssertRCReturn(rc, rc);
2482
2483	/* Update VCPU with the currently set VM-exit controls. */
2484	pVCpu->hm.s.vmx.u32EntryCtls = val;
2485	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2486	}
2487	return rc;
2488	}
2489
2490
2491	/**
2492	* Sets up the VM-exit controls in the VMCS.
2493	*
2494	* @returns VBox status code.
2495	* @param pVM Pointer to the VM.
2496	* @param pVCpu Pointer to the VMCPU.
2497	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2498	* out-of-sync. Make sure to update the required fields
2499	* before using them.
2500	*
2501	* @remarks requires EFER.
2502	*/
2503	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2504	{
2505	int rc = VINF_SUCCESS;
2506	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2507	{
2508	PVM pVM = pVCpu->CTX_SUFF(pVM);
2509	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2510	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2511
2512	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2513	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2514
2515	/*
2516	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2517	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2518	*/
2519	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2520	if (HMVMX_IS_64BIT_HOST_MODE())
2521	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2522	else
2523	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2524	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2525	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2526	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2527	else
2528	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2529	#endif
2530
2531	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2532	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2533
2534	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2535	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2536	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2537	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2538	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2539
2540	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2541	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2542
2543	if ((val & zap) != val)
2544	{
2545	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2546	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2547	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2548	}
2549
2550	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2551	AssertRCReturn(rc, rc);
2552
2553	/* Update VCPU with the currently set VM-exit controls. */
2554	pVCpu->hm.s.vmx.u32ExitCtls = val;
2555	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2556	}
2557	return rc;
2558	}
2559
2560
2561	/**
2562	* Loads the guest APIC and related state.
2563	*
2564	* @returns VBox status code.
2565	* @param pVM Pointer to the VM.
2566	* @param pVCpu Pointer to the VMCPU.
2567	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2568	* out-of-sync. Make sure to update the required fields
2569	* before using them.
2570	*/
2571	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2572	{
2573	int rc = VINF_SUCCESS;
2574	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2575	{
2576	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2577	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2578	{
2579	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2580
2581	bool fPendingIntr = false;
2582	uint8_t u8Tpr = 0;
2583	uint8_t u8PendingIntr = 0;
2584	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2585	AssertRCReturn(rc, rc);
2586
2587	/*
2588	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2589	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2590	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2591	* the interrupt when we VM-exit for other reasons.
2592	*/
2593	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2594	uint32_t u32TprThreshold = 0;
2595	if (fPendingIntr)
2596	{
2597	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2598	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2599	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2600	if (u8PendingPriority <= u8TprPriority)
2601	u32TprThreshold = u8PendingPriority;
2602	else
2603	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2604	}
2605	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2606
2607	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2608	AssertRCReturn(rc, rc);
2609
2610	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2611	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2612	{
2613	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2614	pMixedCtx->msrLSTAR = u8Tpr;
2615	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2616	{
2617	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2618	if (fPendingIntr)
2619	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2620	else
2621	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2622	}
2623	}
2624	}
2625
2626	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2627	}
2628	return rc;
2629	}
2630
2631
2632	/**
2633	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2634	*
2635	* @returns
2636	* @param pVCpu Pointer to the VMCPU.
2637	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2638	* out-of-sync. Make sure to update the required fields
2639	* before using them.
2640	*
2641	* @remarks No-long-jump zone!!!
2642	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2643	*/
2644	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2645	{
2646	/*
2647	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2648	* inhibit interrupts or clear any existing interrupt-inhibition.
2649	*/
2650	uint32_t uIntrState = 0;
2651	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2652	{
2653	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2654	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2655	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2656	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2657	{
2658	/*
2659	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2660	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2661	*/
2662	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2663	}
2664	else if (pMixedCtx->eflags.Bits.u1IF)
2665	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2666	else
2667	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2668	}
2669	return uIntrState;
2670	}
2671
2672
2673	/**
2674	* Loads the guest's interruptibility-state into the guest-state area in the
2675	* VMCS.
2676	*
2677	* @returns VBox status code.
2678	* @param pVCpu Pointer to the VMCPU.
2679	* @param uIntrState The interruptibility-state to set.
2680	*/
2681	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2682	{
2683	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2684	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2685	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2686	AssertRCReturn(rc, rc);
2687	return rc;
2688	}
2689
2690
2691	/**
2692	* Loads the guest's RIP into the guest-state area in the VMCS.
2693	*
2694	* @returns VBox status code.
2695	* @param pVCpu Pointer to the VMCPU.
2696	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2697	* out-of-sync. Make sure to update the required fields
2698	* before using them.
2699	*
2700	* @remarks No-long-jump zone!!!
2701	*/
2702	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2703	{
2704	int rc = VINF_SUCCESS;
2705	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2706	{
2707	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2708	AssertRCReturn(rc, rc);
2709	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2710	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2711	}
2712	return rc;
2713	}
2714
2715
2716	/**
2717	* Loads the guest's RSP into the guest-state area in the VMCS.
2718	*
2719	* @returns VBox status code.
2720	* @param pVCpu Pointer to the VMCPU.
2721	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2722	* out-of-sync. Make sure to update the required fields
2723	* before using them.
2724	*
2725	* @remarks No-long-jump zone!!!
2726	*/
2727	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2728	{
2729	int rc = VINF_SUCCESS;
2730	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2731	{
2732	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2733	AssertRCReturn(rc, rc);
2734	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2735	}
2736	return rc;
2737	}
2738
2739
2740	/**
2741	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2742	*
2743	* @returns VBox status code.
2744	* @param pVCpu Pointer to the VMCPU.
2745	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2746	* out-of-sync. Make sure to update the required fields
2747	* before using them.
2748	*
2749	* @remarks No-long-jump zone!!!
2750	*/
2751	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2752	{
2753	int rc = VINF_SUCCESS;
2754	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2755	{
2756	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2757	Let us assert it as such and use 32-bit VMWRITE. */
2758	Assert(!(pMixedCtx->rflags.u64 >> 32));
2759	X86EFLAGS uEFlags = pMixedCtx->eflags;
2760	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2761	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2762
2763	/*
2764	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2765	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2766	*/
2767	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2768	{
2769	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2770	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2771	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2772	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2773	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2774	}
2775
2776	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2777	AssertRCReturn(rc, rc);
2778
2779	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2780	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2781	}
2782	return rc;
2783	}
2784
2785
2786	/**
2787	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2788	*
2789	* @returns VBox status code.
2790	* @param pVCpu Pointer to the VMCPU.
2791	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2792	* out-of-sync. Make sure to update the required fields
2793	* before using them.
2794	*
2795	* @remarks No-long-jump zone!!!
2796	*/
2797	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2798	{
2799	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2800	AssertRCReturn(rc, rc);
2801	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2802	AssertRCReturn(rc, rc);
2803	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2804	AssertRCReturn(rc, rc);
2805	return rc;
2806	}
2807
2808
2809	/**
2810	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2811	* in the VMCS.
2812	*
2813	* @returns VBox status code.
2814	* @param pVM Pointer to the VM.
2815	* @param pVCpu Pointer to the VMCPU.
2816	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2817	* out-of-sync. Make sure to update the required fields
2818	* before using them.
2819	*
2820	* @remarks No-long-jump zone!!!
2821	*/
2822	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2823	{
2824	int rc = VINF_SUCCESS;
2825	PVM pVM = pVCpu->CTX_SUFF(pVM);
2826
2827	/*
2828	* Guest CR0.
2829	* Guest FPU.
2830	*/
2831	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2832	{
2833	Assert(!(pCtx->cr0 >> 32));
2834	uint32_t u32GuestCR0 = pCtx->cr0;
2835
2836	/* The guest's view (read access) of its CR0 is unblemished. */
2837	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2838	AssertRCReturn(rc, rc);
2839	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2840
2841	/* Setup VT-x's view of the guest CR0. */
2842	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2843	if (pVM->hm.s.fNestedPaging)
2844	{
2845	if (CPUMIsGuestPagingEnabledEx(pCtx))
2846	{
2847	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2848	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2849	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2850	}
2851	else
2852	{
2853	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2854	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2855	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2856	}
2857
2858	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2859	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2860	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2861
2862	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2863	AssertRCReturn(rc, rc);
2864	}
2865	else
2866	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2867
2868	/*
2869	* Guest FPU bits.
2870	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2871	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2872	*/
2873	u32GuestCR0 \|= X86_CR0_NE;
2874	bool fInterceptNM = false;
2875	if (CPUMIsGuestFPUStateActive(pVCpu))
2876	{
2877	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2878	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2879	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2880	}
2881	else
2882	{
2883	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2884	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2885	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2886	}
2887
2888	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2889	bool fInterceptMF = false;
2890	if (!(pCtx->cr0 & X86_CR0_NE))
2891	fInterceptMF = true;
2892
2893	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2894	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2895	{
2896	Assert(PDMVmmDevHeapIsEnabled(pVM));
2897	Assert(pVM->hm.s.vmx.pRealModeTSS);
2898	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2899	fInterceptNM = true;
2900	fInterceptMF = true;
2901	}
2902	else
2903	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2904
2905	if (fInterceptNM)
2906	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2907	else
2908	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2909
2910	if (fInterceptMF)
2911	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2912	else
2913	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2914
2915	/* Additional intercepts for debugging, define these yourself explicitly. */
2916	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2917	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2918	\| RT_BIT(X86_XCPT_DB)
2919	\| RT_BIT(X86_XCPT_DE)
2920	\| RT_BIT(X86_XCPT_NM)
2921	\| RT_BIT(X86_XCPT_UD)
2922	\| RT_BIT(X86_XCPT_NP)
2923	\| RT_BIT(X86_XCPT_SS)
2924	\| RT_BIT(X86_XCPT_GP)
2925	\| RT_BIT(X86_XCPT_PF)
2926	\| RT_BIT(X86_XCPT_MF);
2927	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2928	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2929	#endif
2930
2931	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2932
2933	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2934	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2935	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2936	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2937	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2938	else
2939	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2940
2941	u32GuestCR0 \|= uSetCR0;
2942	u32GuestCR0 &= uZapCR0;
2943	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2944
2945	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2946	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2947	AssertRCReturn(rc, rc);
2948	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2949	AssertRCReturn(rc, rc);
2950	Log(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2951
2952	/*
2953	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2954	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2955	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2956	*/
2957	uint32_t u32CR0Mask = 0;
2958	u32CR0Mask = X86_CR0_PE
2959	\| X86_CR0_NE
2960	\| X86_CR0_WP
2961	\| X86_CR0_PG
2962	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2963	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2964	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2965	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2966	u32CR0Mask &= ~X86_CR0_PE;
2967	if (pVM->hm.s.fNestedPaging)
2968	u32CR0Mask &= ~X86_CR0_WP;
2969
2970	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2971	if (fInterceptNM)
2972	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2973	else
2974	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2975
2976	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2977	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
2978	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
2979	AssertRCReturn(rc, rc);
2980
2981	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2982	}
2983
2984	/*
2985	* Guest CR2.
2986	* It's always loaded in the assembler code. Nothing to do here.
2987	*/
2988
2989	/*
2990	* Guest CR3.
2991	*/
2992	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2993	{
2994	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2995	if (pVM->hm.s.fNestedPaging)
2996	{
2997	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2998
2999	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3000	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3001	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3002	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3003
3004	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3005	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3006	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3007
3008	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3009	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3010	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3011	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3012
3013	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3014	AssertRCReturn(rc, rc);
3015	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3016
3017	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3018	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3019	{
3020	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3021	if (CPUMIsGuestInPAEModeEx(pCtx))
3022	{
3023	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3024	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3025	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3026	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3027	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3028	}
3029
3030	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3031	have Unrestricted Execution to handle the guest when it's not using paging. */
3032	GCPhysGuestCR3 = pCtx->cr3;
3033	}
3034	else
3035	{
3036	/*
3037	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3038	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3039	* EPT takes care of translating it to host-physical addresses.
3040	*/
3041	RTGCPHYS GCPhys;
3042	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3043	Assert(PDMVmmDevHeapIsEnabled(pVM));
3044
3045	/* We obtain it here every time as the guest could have relocated this PCI region. */
3046	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3047	AssertRCReturn(rc, rc);
3048
3049	GCPhysGuestCR3 = GCPhys;
3050	}
3051
3052	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3053	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3054	}
3055	else
3056	{
3057	/* Non-nested paging case, just use the hypervisor's CR3. */
3058	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3059
3060	Log(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3061	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3062	}
3063	AssertRCReturn(rc, rc);
3064
3065	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3066	}
3067
3068	/*
3069	* Guest CR4.
3070	*/
3071	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3072	{
3073	Assert(!(pCtx->cr4 >> 32));
3074	uint32_t u32GuestCR4 = pCtx->cr4;
3075
3076	/* The guest's view of its CR4 is unblemished. */
3077	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3078	AssertRCReturn(rc, rc);
3079	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3080
3081	/* Setup VT-x's view of the guest CR4. */
3082	/*
3083	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3084	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3085	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3086	*/
3087	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3088	{
3089	Assert(pVM->hm.s.vmx.pRealModeTSS);
3090	Assert(PDMVmmDevHeapIsEnabled(pVM));
3091	u32GuestCR4 &= ~X86_CR4_VME;
3092	}
3093
3094	if (pVM->hm.s.fNestedPaging)
3095	{
3096	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3097	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3098	{
3099	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3100	u32GuestCR4 \|= X86_CR4_PSE;
3101	/* Our identity mapping is a 32 bits page directory. */
3102	u32GuestCR4 &= ~X86_CR4_PAE;
3103	}
3104	/* else use guest CR4.*/
3105	}
3106	else
3107	{
3108	/*
3109	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3110	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3111	*/
3112	switch (pVCpu->hm.s.enmShadowMode)
3113	{
3114	case PGMMODE_REAL: /* Real-mode. */
3115	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3116	case PGMMODE_32_BIT: /* 32-bit paging. */
3117	{
3118	u32GuestCR4 &= ~X86_CR4_PAE;
3119	break;
3120	}
3121
3122	case PGMMODE_PAE: /* PAE paging. */
3123	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3124	{
3125	u32GuestCR4 \|= X86_CR4_PAE;
3126	break;
3127	}
3128
3129	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3130	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3131	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3132	break;
3133	#endif
3134	default:
3135	AssertFailed();
3136	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3137	}
3138	}
3139
3140	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3141	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3142	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3143	u32GuestCR4 \|= uSetCR4;
3144	u32GuestCR4 &= uZapCR4;
3145
3146	/* Write VT-x's view of the guest CR4 into the VMCS. */
3147	Log(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3148	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3149	AssertRCReturn(rc, rc);
3150
3151	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3152	uint32_t u32CR4Mask = 0;
3153	u32CR4Mask = X86_CR4_VME
3154	\| X86_CR4_PAE
3155	\| X86_CR4_PGE
3156	\| X86_CR4_PSE
3157	\| X86_CR4_VMXE;
3158	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3159	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3160	AssertRCReturn(rc, rc);
3161
3162	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3163	}
3164	return rc;
3165	}
3166
3167
3168	/**
3169	* Loads the guest debug registers into the guest-state area in the VMCS.
3170	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3171	*
3172	* @returns VBox status code.
3173	* @param pVCpu Pointer to the VMCPU.
3174	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3175	* out-of-sync. Make sure to update the required fields
3176	* before using them.
3177	*
3178	* @remarks No-long-jump zone!!!
3179	*/
3180	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3181	{
3182	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3183	return VINF_SUCCESS;
3184
3185	#ifdef VBOX_STRICT
3186	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3187	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3188	{
3189	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3190	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3191	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3192	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3193	}
3194	#endif
3195
3196	int rc = VERR_INTERNAL_ERROR_5;
3197	PVM pVM = pVCpu->CTX_SUFF(pVM);
3198	bool fInterceptDB = false;
3199	bool fInterceptMovDRx = false;
3200	if (DBGFIsStepping(pVCpu))
3201	{
3202	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3203	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3204	{
3205	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3206	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3207	AssertRCReturn(rc, rc);
3208	Assert(fInterceptDB == false);
3209	}
3210	else
3211	{
3212	fInterceptDB = true;
3213	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3214	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3215	}
3216	}
3217
3218	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3219	{
3220	if (!CPUMIsHyperDebugStateActive(pVCpu))
3221	{
3222	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3223	AssertRC(rc);
3224	}
3225	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3226	fInterceptMovDRx = true;
3227	}
3228	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3229	{
3230	if (!CPUMIsGuestDebugStateActive(pVCpu))
3231	{
3232	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3233	AssertRC(rc);
3234	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3235	}
3236	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3237	Assert(fInterceptMovDRx == false);
3238	}
3239	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3240	{
3241	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3242	fInterceptMovDRx = true;
3243	}
3244
3245	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3246	if (fInterceptDB)
3247	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3248	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3249	{
3250	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3251	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3252	#endif
3253	}
3254
3255	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3256	if (fInterceptMovDRx)
3257	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3258	else
3259	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3260
3261	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3262	AssertRCReturn(rc, rc);
3263	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3264	AssertRCReturn(rc, rc);
3265
3266	/* The guest's view of its DR7 is unblemished. Use 32-bit write as upper 32-bits MBZ as asserted above. */
3267	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
3268	AssertRCReturn(rc, rc);
3269
3270	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3271	return rc;
3272	}
3273
3274
3275	#ifdef VBOX_STRICT
3276	/**
3277	* Strict function to validate segment registers.
3278	*
3279	* @remarks Requires CR0.
3280	*/
3281	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3282	{
3283	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3284	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3285	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3286
3287	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3288	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3289	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3290	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3291	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3292	{
3293	/* Protected mode checks */
3294	/* CS */
3295	Assert(pCtx->cs.Attr.n.u1Present);
3296	Assert(!(pCtx->cs.Attr.u & 0xf00));
3297	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3298	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3299	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3300	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3301	\|\| (pCtx->cs.Attr.n.u1Granularity));
3302	/* CS cannot be loaded with NULL in protected mode. */
3303	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3304	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3305	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3306	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3307	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3308	else
3309	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3310	/* SS */
3311	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3312	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3313	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3314	if ( !(pCtx->cr0 & X86_CR0_PE)
3315	\|\| pCtx->cs.Attr.n.u4Type == 3)
3316	{
3317	Assert(!pCtx->ss.Attr.n.u2Dpl);
3318	}
3319	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3320	{
3321	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3322	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3323	Assert(pCtx->ss.Attr.n.u1Present);
3324	Assert(!(pCtx->ss.Attr.u & 0xf00));
3325	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3326	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3327	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3328	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3329	\|\| (pCtx->ss.Attr.n.u1Granularity));
3330	}
3331	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3332	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3333	{
3334	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3335	Assert(pCtx->ds.Attr.n.u1Present);
3336	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3337	Assert(!(pCtx->ds.Attr.u & 0xf00));
3338	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3339	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3340	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3341	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3342	\|\| (pCtx->ds.Attr.n.u1Granularity));
3343	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3344	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3345	}
3346	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3347	{
3348	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3349	Assert(pCtx->es.Attr.n.u1Present);
3350	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3351	Assert(!(pCtx->es.Attr.u & 0xf00));
3352	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3353	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3354	\|\| !(pCtx->es.Attr.n.u1Granularity));
3355	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3356	\|\| (pCtx->es.Attr.n.u1Granularity));
3357	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3358	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3359	}
3360	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3361	{
3362	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3363	Assert(pCtx->fs.Attr.n.u1Present);
3364	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3365	Assert(!(pCtx->fs.Attr.u & 0xf00));
3366	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3367	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3368	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3369	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3370	\|\| (pCtx->fs.Attr.n.u1Granularity));
3371	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3372	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3373	}
3374	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3375	{
3376	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3377	Assert(pCtx->gs.Attr.n.u1Present);
3378	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3379	Assert(!(pCtx->gs.Attr.u & 0xf00));
3380	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3381	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3382	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3383	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3384	\|\| (pCtx->gs.Attr.n.u1Granularity));
3385	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3386	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3387	}
3388	/* 64-bit capable CPUs. */
3389	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3390	Assert(!(pCtx->cs.u64Base >> 32));
3391	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3392	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3393	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3394	# endif
3395	}
3396	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3397	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3398	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3399	{
3400	/* Real and v86 mode checks. */
3401	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3402	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3403	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3404	{
3405	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3406	}
3407	else
3408	{
3409	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3410	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3411	}
3412
3413	/* CS */
3414	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3415	Assert(pCtx->cs.u32Limit == 0xffff);
3416	Assert(u32CSAttr == 0xf3);
3417	/* SS */
3418	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3419	Assert(pCtx->ss.u32Limit == 0xffff);
3420	Assert(u32SSAttr == 0xf3);
3421	/* DS */
3422	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3423	Assert(pCtx->ds.u32Limit == 0xffff);
3424	Assert(u32DSAttr == 0xf3);
3425	/* ES */
3426	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3427	Assert(pCtx->es.u32Limit == 0xffff);
3428	Assert(u32ESAttr == 0xf3);
3429	/* FS */
3430	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3431	Assert(pCtx->fs.u32Limit == 0xffff);
3432	Assert(u32FSAttr == 0xf3);
3433	/* GS */
3434	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3435	Assert(pCtx->gs.u32Limit == 0xffff);
3436	Assert(u32GSAttr == 0xf3);
3437	/* 64-bit capable CPUs. */
3438	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3439	Assert(!(pCtx->cs.u64Base >> 32));
3440	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3441	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3442	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3443	# endif
3444	}
3445	}
3446	#endif /* VBOX_STRICT */
3447
3448
3449	/**
3450	* Writes a guest segment register into the guest-state area in the VMCS.
3451	*
3452	* @returns VBox status code.
3453	* @param pVCpu Pointer to the VMCPU.
3454	* @param idxSel Index of the selector in the VMCS.
3455	* @param idxLimit Index of the segment limit in the VMCS.
3456	* @param idxBase Index of the segment base in the VMCS.
3457	* @param idxAccess Index of the access rights of the segment in the VMCS.
3458	* @param pSelReg Pointer to the segment selector.
3459	* @param pCtx Pointer to the guest-CPU context.
3460	*
3461	* @remarks No-long-jump zone!!!
3462	*/
3463	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3464	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3465	{
3466	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3467	AssertRCReturn(rc, rc);
3468	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3469	AssertRCReturn(rc, rc);
3470	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3471	AssertRCReturn(rc, rc);
3472
3473	uint32_t u32Access = pSelReg->Attr.u;
3474	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3475	{
3476	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3477	u32Access = 0xf3;
3478	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3479	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3480	}
3481	else
3482	{
3483	/*
3484	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3485	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3486	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3487	* loaded in protected-mode have their attribute as 0.
3488	*/
3489	if (!u32Access)
3490	u32Access = HMVMX_SEL_UNUSABLE;
3491	}
3492
3493	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3494	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3495	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3496
3497	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3498	AssertRCReturn(rc, rc);
3499	return rc;
3500	}
3501
3502
3503	/**
3504	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3505	* into the guest-state area in the VMCS.
3506	*
3507	* @returns VBox status code.
3508	* @param pVM Pointer to the VM.
3509	* @param pVCPU Pointer to the VMCPU.
3510	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3511	* out-of-sync. Make sure to update the required fields
3512	* before using them.
3513	*
3514	* @remarks Requires CR0 (strict builds validation).
3515	* @remarks No-long-jump zone!!!
3516	*/
3517	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3518	{
3519	int rc = VERR_INTERNAL_ERROR_5;
3520	PVM pVM = pVCpu->CTX_SUFF(pVM);
3521
3522	/*
3523	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3524	*/
3525	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3526	{
3527	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3528	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3529	{
3530	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3531	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3532	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3533	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3534	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3535	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3536	}
3537
3538	#ifdef VBOX_WITH_REM
3539	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3540	{
3541	Assert(pVM->hm.s.vmx.pRealModeTSS);
3542	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3543	if ( pVCpu->hm.s.vmx.fWasInRealMode
3544	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3545	{
3546	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3547	in real-mode (e.g. OpenBSD 4.0) */
3548	REMFlushTBs(pVM);
3549	Log(("Load: Switch to protected mode detected!\n"));
3550	pVCpu->hm.s.vmx.fWasInRealMode = false;
3551	}
3552	}
3553	#endif
3554	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3555	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3556	AssertRCReturn(rc, rc);
3557	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3558	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3559	AssertRCReturn(rc, rc);
3560	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3561	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3562	AssertRCReturn(rc, rc);
3563	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3564	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3565	AssertRCReturn(rc, rc);
3566	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3567	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3568	AssertRCReturn(rc, rc);
3569	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3570	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3571	AssertRCReturn(rc, rc);
3572
3573	Log(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3574	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3575	#ifdef VBOX_STRICT
3576	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3577	#endif
3578	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3579	}
3580
3581	/*
3582	* Guest TR.
3583	*/
3584	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3585	{
3586	/*
3587	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3588	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3589	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3590	*/
3591	uint16_t u16Sel = 0;
3592	uint32_t u32Limit = 0;
3593	uint64_t u64Base = 0;
3594	uint32_t u32AccessRights = 0;
3595
3596	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3597	{
3598	u16Sel = pMixedCtx->tr.Sel;
3599	u32Limit = pMixedCtx->tr.u32Limit;
3600	u64Base = pMixedCtx->tr.u64Base;
3601	u32AccessRights = pMixedCtx->tr.Attr.u;
3602	}
3603	else
3604	{
3605	Assert(pVM->hm.s.vmx.pRealModeTSS);
3606	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3607
3608	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3609	RTGCPHYS GCPhys;
3610	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3611	AssertRCReturn(rc, rc);
3612
3613	X86DESCATTR DescAttr;
3614	DescAttr.u = 0;
3615	DescAttr.n.u1Present = 1;
3616	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3617
3618	u16Sel = 0;
3619	u32Limit = HM_VTX_TSS_SIZE;
3620	u64Base = GCPhys; /* in real-mode phys = virt. */
3621	u32AccessRights = DescAttr.u;
3622	}
3623
3624	/* Validate. */
3625	Assert(!(u16Sel & RT_BIT(2)));
3626	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3627	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3628	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3629	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3630	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3631	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3632	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3633	Assert( (u32Limit & 0xfff) == 0xfff
3634	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3635	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3636	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3637
3638	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3639	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3640	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3641	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3642
3643	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3644	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3645	}
3646
3647	/*
3648	* Guest GDTR.
3649	*/
3650	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3651	{
3652	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3653	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3654
3655	Assert(!(pMixedCtx->gdtr.cbGdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3656	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3657	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3658	}
3659
3660	/*
3661	* Guest LDTR.
3662	*/
3663	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3664	{
3665	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3666	uint32_t u32Access = 0;
3667	if (!pMixedCtx->ldtr.Attr.u)
3668	u32Access = HMVMX_SEL_UNUSABLE;
3669	else
3670	u32Access = pMixedCtx->ldtr.Attr.u;
3671
3672	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3673	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3674	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3675	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3676
3677	/* Validate. */
3678	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3679	{
3680	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3681	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3682	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3683	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3684	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3685	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3686	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3687	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3688	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3689	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3690	}
3691
3692	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3693	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3694	}
3695
3696	/*
3697	* Guest IDTR.
3698	*/
3699	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3700	{
3701	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3702	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3703
3704	Assert(!(pMixedCtx->idtr.cbIdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3705	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3706	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3707	}
3708
3709	return VINF_SUCCESS;
3710	}
3711
3712
3713	/**
3714	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3715	* areas. These MSRs will automatically be loaded to the host CPU on every
3716	* successful VM entry and stored from the host CPU on every successful VM exit.
3717	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3718	*
3719	* @returns VBox status code.
3720	* @param pVCpu Pointer to the VMCPU.
3721	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3722	* out-of-sync. Make sure to update the required fields
3723	* before using them.
3724	*
3725	* @remarks No-long-jump zone!!!
3726	*/
3727	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3728	{
3729	AssertPtr(pVCpu);
3730	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3731
3732	/*
3733	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3734	*/
3735	int rc = VINF_SUCCESS;
3736	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3737	{
3738	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3739	PVM pVM = pVCpu->CTX_SUFF(pVM);
3740	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3741	uint32_t cGuestMsrs = 0;
3742
3743	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3744	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3745	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3746	if (fSupportsNX \|\| fSupportsLongMode)
3747	{
3748	/** @todo support save IA32_EFER, i.e.
3749	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, in which case the
3750	* guest EFER need not be part of the VM-entry MSR-load area. Also
3751	* allow the guest to read EFER without causing a VM-exit when
3752	* possible. */
3753	/* Do -not- load guest EFER as we don't save/restore the host EFER always. See hmr0VmxSaveHostMsrs() */
3754	#if 0
3755	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3756	pGuestMsr->u32Reserved = 0;
3757	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3758	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3759	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3760	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3761	pGuestMsr++; cGuestMsrs++;
3762	#endif
3763	if (fSupportsLongMode)
3764	{
3765	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3766	pGuestMsr->u32Reserved = 0;
3767	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3768	pGuestMsr++; cGuestMsrs++;
3769	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3770	pGuestMsr->u32Reserved = 0;
3771	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3772	pGuestMsr++; cGuestMsrs++;
3773	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3774	pGuestMsr->u32Reserved = 0;
3775	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3776	pGuestMsr++; cGuestMsrs++;
3777	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3778	pGuestMsr->u32Reserved = 0;
3779	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3780	pGuestMsr++; cGuestMsrs++;
3781	}
3782	}
3783
3784	/*
3785	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3786	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3787	*/
3788	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3789	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3790	{
3791	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3792	pGuestMsr->u32Reserved = 0;
3793	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3794	AssertRCReturn(rc, rc);
3795	pGuestMsr++; cGuestMsrs++;
3796	}
3797
3798	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3799	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3800	{
3801	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3802	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3803	}
3804
3805	/* Update the VCPU's copy of the guest MSR count. */
3806	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3807	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3808	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3809	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3810
3811	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3812	}
3813
3814	/*
3815	* Guest Sysenter MSRs.
3816	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3817	* VM-exits on WRMSRs for these MSRs.
3818	*/
3819	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3820	{
3821	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3822	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3823	}
3824	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3825	{
3826	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3827	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3828	}
3829	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3830	{
3831	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3832	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3833	}
3834
3835	return rc;
3836	}
3837
3838
3839	/**
3840	* Loads the guest activity state into the guest-state area in the VMCS.
3841	*
3842	* @returns VBox status code.
3843	* @param pVCpu Pointer to the VMCPU.
3844	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3845	* out-of-sync. Make sure to update the required fields
3846	* before using them.
3847	*
3848	* @remarks No-long-jump zone!!!
3849	*/
3850	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3851	{
3852	/** @todo See if we can make use of other states, e.g.
3853	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3854	int rc = VINF_SUCCESS;
3855	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3856	{
3857	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3858	AssertRCReturn(rc, rc);
3859	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3860	}
3861	return rc;
3862	}
3863
3864
3865	/**
3866	* Sets up the appropriate function to run guest code.
3867	*
3868	* @returns VBox status code.
3869	* @param pVCpu Pointer to the VMCPU.
3870	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3871	* out-of-sync. Make sure to update the required fields
3872	* before using them.
3873	*
3874	* @remarks No-long-jump zone!!!
3875	*/
3876	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3877	{
3878	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3879	{
3880	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3881	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3882	#endif
3883	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3884	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3885	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3886	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3887	#else
3888	/* 64-bit host or hybrid host. */
3889	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3890	#endif
3891	}
3892	else
3893	{
3894	/* Guest is not in long mode, use the 32-bit handler. */
3895	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3896	}
3897	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3898	return VINF_SUCCESS;
3899	}
3900
3901
3902	/**
3903	* Wrapper for running the guest code in VT-x.
3904	*
3905	* @returns VBox strict status code.
3906	* @param pVM Pointer to the VM.
3907	* @param pVCpu Pointer to the VMCPU.
3908	* @param pCtx Pointer to the guest-CPU context.
3909	*
3910	* @remarks No-long-jump zone!!!
3911	*/
3912	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3913	{
3914	/*
3915	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3916	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3917	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3918	*/
3919	#ifdef VBOX_WITH_KERNEL_USING_XMM
3920	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3921	#else
3922	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3923	#endif
3924	}
3925
3926
3927	/**
3928	* Report world-switch error and dump some useful debug info.
3929	*
3930	* @param pVM Pointer to the VM.
3931	* @param pVCpu Pointer to the VMCPU.
3932	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3933	* @param pCtx Pointer to the guest-CPU context.
3934	* @param pVmxTransient Pointer to the VMX transient structure (only
3935	* exitReason updated).
3936	*/
3937	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3938	{
3939	Assert(pVM);
3940	Assert(pVCpu);
3941	Assert(pCtx);
3942	Assert(pVmxTransient);
3943	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3944
3945	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3946	switch (rcVMRun)
3947	{
3948	case VERR_VMX_INVALID_VMXON_PTR:
3949	AssertFailed();
3950	break;
3951	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3952	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3953	{
3954	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3955	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3956	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3957	AssertRC(rc);
3958
3959	#ifdef VBOX_STRICT
3960	Log(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3961	pVmxTransient->uExitReason));
3962	Log(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
3963	Log(("InstrError %#RX32\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3964	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3965	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3966	else
3967	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3968
3969	/* VMX control bits. */
3970	uint32_t u32Val;
3971	uint64_t u64Val;
3972	HMVMXHCUINTREG uHCReg;
3973	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
3974	Log(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
3975	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
3976	Log(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
3977	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
3978	Log(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
3979	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
3980	Log(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
3981	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
3982	Log(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
3983	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3984	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3985	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3986	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3987	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3988	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3989	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3990	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3991	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3992	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3993	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3994	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3995	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3996	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3997	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3998	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3999	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4000	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4001	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4002	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4003	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4004	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4005	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4006	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4007	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4008	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4009	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4010	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4011	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4012	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4013	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4014	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4015
4016	/* Guest bits. */
4017	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4018	Log(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4019	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4020	Log(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4021	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4022	Log(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4023	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4024	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4025
4026	/* Host bits. */
4027	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4028	Log(("Host CR0 %#RHr\n", uHCReg));
4029	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4030	Log(("Host CR3 %#RHr\n", uHCReg));
4031	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4032	Log(("Host CR4 %#RHr\n", uHCReg));
4033
4034	RTGDTR HostGdtr;
4035	PCX86DESCHC pDesc;
4036	ASMGetGDTR(&HostGdtr);
4037	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4038	Log(("Host CS %#08x\n", u32Val));
4039	if (u32Val < HostGdtr.cbGdt)
4040	{
4041	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4042	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4043	}
4044
4045	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4046	Log(("Host DS %#08x\n", u32Val));
4047	if (u32Val < HostGdtr.cbGdt)
4048	{
4049	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4050	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4051	}
4052
4053	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4054	Log(("Host ES %#08x\n", u32Val));
4055	if (u32Val < HostGdtr.cbGdt)
4056	{
4057	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4058	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4059	}
4060
4061	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4062	Log(("Host FS %#08x\n", u32Val));
4063	if (u32Val < HostGdtr.cbGdt)
4064	{
4065	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4066	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4067	}
4068
4069	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4070	Log(("Host GS %#08x\n", u32Val));
4071	if (u32Val < HostGdtr.cbGdt)
4072	{
4073	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4074	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4075	}
4076
4077	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4078	Log(("Host SS %#08x\n", u32Val));
4079	if (u32Val < HostGdtr.cbGdt)
4080	{
4081	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4082	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4083	}
4084
4085	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4086	Log(("Host TR %#08x\n", u32Val));
4087	if (u32Val < HostGdtr.cbGdt)
4088	{
4089	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4090	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4091	}
4092
4093	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4094	Log(("Host TR Base %#RHv\n", uHCReg));
4095	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4096	Log(("Host GDTR Base %#RHv\n", uHCReg));
4097	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4098	Log(("Host IDTR Base %#RHv\n", uHCReg));
4099	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4100	Log(("Host SYSENTER CS %#08x\n", u32Val));
4101	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4102	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
4103	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4104	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
4105	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4106	Log(("Host RSP %#RHv\n", uHCReg));
4107	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4108	Log(("Host RIP %#RHv\n", uHCReg));
4109	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4110	if (HMVMX_IS_64BIT_HOST_MODE())
4111	{
4112	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4113	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4114	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4115	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4116	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4117	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4118	}
4119	# endif
4120	#endif /* VBOX_STRICT */
4121	break;
4122	}
4123
4124	default:
4125	/* Impossible */
4126	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4127	break;
4128	}
4129	NOREF(pVM);
4130	}
4131
4132
4133	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4134	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4135	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4136	#endif
4137	#ifdef VBOX_STRICT
4138	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4139	{
4140	switch (idxField)
4141	{
4142	case VMX_VMCS_GUEST_RIP:
4143	case VMX_VMCS_GUEST_RSP:
4144	case VMX_VMCS_GUEST_SYSENTER_EIP:
4145	case VMX_VMCS_GUEST_SYSENTER_ESP:
4146	case VMX_VMCS_GUEST_GDTR_BASE:
4147	case VMX_VMCS_GUEST_IDTR_BASE:
4148	case VMX_VMCS_GUEST_CS_BASE:
4149	case VMX_VMCS_GUEST_DS_BASE:
4150	case VMX_VMCS_GUEST_ES_BASE:
4151	case VMX_VMCS_GUEST_FS_BASE:
4152	case VMX_VMCS_GUEST_GS_BASE:
4153	case VMX_VMCS_GUEST_SS_BASE:
4154	case VMX_VMCS_GUEST_LDTR_BASE:
4155	case VMX_VMCS_GUEST_TR_BASE:
4156	case VMX_VMCS_GUEST_CR3:
4157	return true;
4158	}
4159	return false;
4160	}
4161
4162	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4163	{
4164	switch (idxField)
4165	{
4166	/* Read-only fields. */
4167	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4168	return true;
4169	}
4170	/* Remaining readable fields should also be writable. */
4171	return hmR0VmxIsValidWriteField(idxField);
4172	}
4173	#endif /* VBOX_STRICT */
4174
4175
4176	/**
4177	* Executes the specified handler in 64-bit mode.
4178	*
4179	* @returns VBox status code.
4180	* @param pVM Pointer to the VM.
4181	* @param pVCpu Pointer to the VMCPU.
4182	* @param pCtx Pointer to the guest CPU context.
4183	* @param enmOp The operation to perform.
4184	* @param cbParam Number of parameters.
4185	* @param paParam Array of 32-bit parameters.
4186	*/
4187	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4188	uint32_t *paParam)
4189	{
4190	int rc, rc2;
4191	PHMGLOBLCPUINFO pCpu;
4192	RTHCPHYS HCPhysCpuPage;
4193	RTCCUINTREG uOldEFlags;
4194
4195	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4196	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4197	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4198	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4199
4200	#ifdef VBOX_STRICT
4201	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4202	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4203
4204	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4205	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4206	#endif
4207
4208	/* Disable interrupts. */
4209	uOldEFlags = ASMIntDisableFlags();
4210
4211	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4212	RTCPUID idHostCpu = RTMpCpuId();
4213	CPUMR0SetLApic(pVM, idHostCpu);
4214	#endif
4215
4216	pCpu = HMR0GetCurrentCpu();
4217	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4218
4219	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4220	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4221
4222	/* Leave VMX Root Mode. */
4223	VMXDisable();
4224
4225	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4226
4227	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4228	CPUMSetHyperEIP(pVCpu, enmOp);
4229	for (int i = (int)cbParam - 1; i >= 0; i--)
4230	CPUMPushHyper(pVCpu, paParam[i]);
4231
4232	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4233
4234	/* Call the switcher. */
4235	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4236	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4237
4238	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4239	/* Make sure the VMX instructions don't cause #UD faults. */
4240	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4241
4242	/* Re-enter VMX Root Mode */
4243	rc2 = VMXEnable(HCPhysCpuPage);
4244	if (RT_FAILURE(rc2))
4245	{
4246	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4247	ASMSetFlags(uOldEFlags);
4248	return rc2;
4249	}
4250
4251	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4252	AssertRC(rc2);
4253	Assert(!(ASMGetFlags() & X86_EFL_IF));
4254	ASMSetFlags(uOldEFlags);
4255	return rc;
4256	}
4257
4258
4259	/**
4260	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4261	* supporting 64-bit guests.
4262	*
4263	* @returns VBox status code.
4264	* @param fResume Whether to VMLAUNCH or VMRESUME.
4265	* @param pCtx Pointer to the guest-CPU context.
4266	* @param pCache Pointer to the VMCS cache.
4267	* @param pVM Pointer to the VM.
4268	* @param pVCpu Pointer to the VMCPU.
4269	*/
4270	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4271	{
4272	uint32_t aParam[6];
4273	PHMGLOBLCPUINFO pCpu = NULL;
4274	RTHCPHYS HCPhysCpuPage = 0;
4275	int rc = VERR_INTERNAL_ERROR_5;
4276
4277	pCpu = HMR0GetCurrentCpu();
4278	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4279
4280	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4281	pCache->uPos = 1;
4282	pCache->interPD = PGMGetInterPaeCR3(pVM);
4283	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4284	#endif
4285
4286	#ifdef VBOX_STRICT
4287	pCache->TestIn.HCPhysCpuPage = 0;
4288	pCache->TestIn.HCPhysVmcs = 0;
4289	pCache->TestIn.pCache = 0;
4290	pCache->TestOut.HCPhysVmcs = 0;
4291	pCache->TestOut.pCache = 0;
4292	pCache->TestOut.pCtx = 0;
4293	pCache->TestOut.eflags = 0;
4294	#endif
4295
4296	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4297	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4298	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4299	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4300	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4301	aParam[5] = 0;
4302
4303	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4304	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4305	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4306	#endif
4307	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4308
4309	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4310	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4311	Assert(pCtx->dr[4] == 10);
4312	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4313	#endif
4314
4315	#ifdef VBOX_STRICT
4316	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4317	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4318	pVCpu->hm.s.vmx.HCPhysVmcs));
4319	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4320	pCache->TestOut.HCPhysVmcs));
4321	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4322	pCache->TestOut.pCache));
4323	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4324	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4325	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4326	pCache->TestOut.pCtx));
4327	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4328	#endif
4329	return rc;
4330	}
4331
4332
4333	/**
4334	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4335	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4336	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4337	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4338	*
4339	* @returns VBox status code.
4340	* @param pVM Pointer to the VM.
4341	* @param pVCpu Pointer to the VMCPU.
4342	*/
4343	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4344	{
4345	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4346	{ \
4347	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4348	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4349	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4350	++cReadFields; \
4351	}
4352
4353	AssertPtr(pVM);
4354	AssertPtr(pVCpu);
4355	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4356	uint32_t cReadFields = 0;
4357
4358	/*
4359	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4360	* and serve to indicate exceptions to the rules.
4361	*/
4362
4363	/* Guest-natural selector base fields. */
4364	#if 0
4365	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4366	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4367	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4368	#endif
4369	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4370	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4371	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4372	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4373	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4374	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4375	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4376	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4377	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4378	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4379	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4380	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4381	#if 0
4382	/* Unused natural width guest-state fields. */
4383	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4384	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4385	#endif
4386	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4387	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4388
4389	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4390	#if 0
4391	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4392	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4393	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4394	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4395	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4396	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4397	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4398	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4399	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4400	#endif
4401
4402	/* Natural width guest-state fields. */
4403	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4404	#if 0
4405	/* Currently unused field. */
4406	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4407	#endif
4408
4409	if (pVM->hm.s.fNestedPaging)
4410	{
4411	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4412	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4413	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4414	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4415	}
4416	else
4417	{
4418	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4419	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4420	}
4421
4422	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4423	return VINF_SUCCESS;
4424	}
4425
4426
4427	/**
4428	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4429	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4430	* darwin, running 64-bit guests).
4431	*
4432	* @returns VBox status code.
4433	* @param pVCpu Pointer to the VMCPU.
4434	* @param idxField The VMCS field encoding.
4435	* @param u64Val 16, 32 or 64 bits value.
4436	*/
4437	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4438	{
4439	int rc;
4440	switch (idxField)
4441	{
4442	/*
4443	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4444	*/
4445	/* 64-bit Control fields. */
4446	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4447	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4448	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4449	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4450	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4451	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4452	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4453	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4454	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4455	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4456	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4457	case VMX_VMCS64_CTRL_EPTP_FULL:
4458	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4459	/* 64-bit Guest-state fields. */
4460	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4461	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4462	case VMX_VMCS64_GUEST_PAT_FULL:
4463	case VMX_VMCS64_GUEST_EFER_FULL:
4464	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4465	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4466	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4467	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4468	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4469	/* 64-bit Host-state fields. */
4470	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4471	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4472	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4473	{
4474	rc = VMXWriteVmcs32(idxField, u64Val);
4475	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4476	break;
4477	}
4478
4479	/*
4480	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4481	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4482	*/
4483	/* Natural-width Guest-state fields. */
4484	case VMX_VMCS_GUEST_CR3:
4485	case VMX_VMCS_GUEST_ES_BASE:
4486	case VMX_VMCS_GUEST_CS_BASE:
4487	case VMX_VMCS_GUEST_SS_BASE:
4488	case VMX_VMCS_GUEST_DS_BASE:
4489	case VMX_VMCS_GUEST_FS_BASE:
4490	case VMX_VMCS_GUEST_GS_BASE:
4491	case VMX_VMCS_GUEST_LDTR_BASE:
4492	case VMX_VMCS_GUEST_TR_BASE:
4493	case VMX_VMCS_GUEST_GDTR_BASE:
4494	case VMX_VMCS_GUEST_IDTR_BASE:
4495	case VMX_VMCS_GUEST_RSP:
4496	case VMX_VMCS_GUEST_RIP:
4497	case VMX_VMCS_GUEST_SYSENTER_ESP:
4498	case VMX_VMCS_GUEST_SYSENTER_EIP:
4499	{
4500	if (!(u64Val >> 32))
4501	{
4502	/* If this field is 64-bit, VT-x will zero out the top bits. */
4503	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4504	}
4505	else
4506	{
4507	/* Assert that only the 32->64 switcher case should ever come here. */
4508	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4509	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4510	}
4511	break;
4512	}
4513
4514	default:
4515	{
4516	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4517	rc = VERR_INVALID_PARAMETER;
4518	break;
4519	}
4520	}
4521	AssertRCReturn(rc, rc);
4522	return rc;
4523	}
4524
4525
4526	/**
4527	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4528	* hosts (except darwin) for 64-bit guests.
4529	*
4530	* @param pVCpu Pointer to the VMCPU.
4531	* @param idxField The VMCS field encoding.
4532	* @param u64Val 16, 32 or 64 bits value.
4533	*/
4534	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4535	{
4536	AssertPtr(pVCpu);
4537	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4538
4539	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4540	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4541
4542	/* Make sure there are no duplicates. */
4543	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4544	{
4545	if (pCache->Write.aField[i] == idxField)
4546	{
4547	pCache->Write.aFieldVal[i] = u64Val;
4548	return VINF_SUCCESS;
4549	}
4550	}
4551
4552	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4553	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4554	pCache->Write.cValidEntries++;
4555	return VINF_SUCCESS;
4556	}
4557
4558	/* Enable later when the assembly code uses these as callbacks. */
4559	#if 0
4560	/*
4561	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4562	*
4563	* @param pVCpu Pointer to the VMCPU.
4564	* @param pCache Pointer to the VMCS cache.
4565	*
4566	* @remarks No-long-jump zone!!!
4567	*/
4568	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4569	{
4570	AssertPtr(pCache);
4571	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4572	{
4573	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4574	AssertRC(rc);
4575	}
4576	pCache->Write.cValidEntries = 0;
4577	}
4578
4579
4580	/**
4581	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4582	*
4583	* @param pVCpu Pointer to the VMCPU.
4584	* @param pCache Pointer to the VMCS cache.
4585	*
4586	* @remarks No-long-jump zone!!!
4587	*/
4588	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4589	{
4590	AssertPtr(pCache);
4591	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4592	{
4593	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4594	AssertRC(rc);
4595	}
4596	}
4597	#endif
4598	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4599
4600
4601	/**
4602	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4603	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4604	* timer.
4605	*
4606	* @returns VBox status code.
4607	* @param pVCpu Pointer to the VMCPU.
4608	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4609	* out-of-sync. Make sure to update the required fields
4610	* before using them.
4611	* @remarks No-long-jump zone!!!
4612	*/
4613	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4614	{
4615	int rc = VERR_INTERNAL_ERROR_5;
4616	bool fOffsettedTsc = false;
4617	PVM pVM = pVCpu->CTX_SUFF(pVM);
4618	if (pVM->hm.s.vmx.fUsePreemptTimer)
4619	{
4620	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4621
4622	/* Make sure the returned values have sane upper and lower boundaries. */
4623	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4624	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4625	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4626	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4627
4628	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4629	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4630	}
4631	else
4632	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4633
4634	if (fOffsettedTsc)
4635	{
4636	uint64_t u64CurTSC = ASMReadTSC();
4637	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4638	{
4639	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4640	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4641
4642	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4643	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4644	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4645	}
4646	else
4647	{
4648	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4649	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4650	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4651	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4652	}
4653	}
4654	else
4655	{
4656	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4657	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4658	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4659	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4660	}
4661	}
4662
4663
4664	/**
4665	* Determines if an exception is a contributory exception. Contributory
4666	* exceptions are ones which can cause double-faults. Page-fault is
4667	* intentionally not included here as it's a conditional contributory exception.
4668	*
4669	* @returns true if the exception is contributory, false otherwise.
4670	* @param uVector The exception vector.
4671	*/
4672	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4673	{
4674	switch (uVector)
4675	{
4676	case X86_XCPT_GP:
4677	case X86_XCPT_SS:
4678	case X86_XCPT_NP:
4679	case X86_XCPT_TS:
4680	case X86_XCPT_DE:
4681	return true;
4682	default:
4683	break;
4684	}
4685	return false;
4686	}
4687
4688
4689	/**
4690	* Sets an event as a pending event to be injected into the guest.
4691	*
4692	* @param pVCpu Pointer to the VMCPU.
4693	* @param u32IntrInfo The VM-entry interruption-information field.
4694	* @param cbInstr The VM-entry instruction length in bytes (for software
4695	* interrupts, exceptions and privileged software
4696	* exceptions).
4697	* @param u32ErrCode The VM-entry exception error code.
4698	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4699	* page-fault.
4700	*/
4701	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4702	RTGCUINTPTR GCPtrFaultAddress)
4703	{
4704	Assert(!pVCpu->hm.s.Event.fPending);
4705	pVCpu->hm.s.Event.fPending = true;
4706	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4707	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4708	pVCpu->hm.s.Event.cbInstr = cbInstr;
4709	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4710	}
4711
4712
4713	/**
4714	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4715	*
4716	* @param pVCpu Pointer to the VMCPU.
4717	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4718	* out-of-sync. Make sure to update the required fields
4719	* before using them.
4720	*/
4721	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4722	{
4723	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4724	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4725	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4726	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4727	}
4728
4729
4730	/**
4731	* Handle a condition that occurred while delivering an event through the guest
4732	* IDT.
4733	*
4734	* @returns VBox status code (informational error codes included).
4735	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4736	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4737	* continue execution of the guest which will delivery the #DF.
4738	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4739	*
4740	* @param pVCpu Pointer to the VMCPU.
4741	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4742	* out-of-sync. Make sure to update the required fields
4743	* before using them.
4744	* @param pVmxTransient Pointer to the VMX transient structure.
4745	*
4746	* @remarks No-long-jump zone!!!
4747	*/
4748	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4749	{
4750	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4751	AssertRC(rc);
4752	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4753	{
4754	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4755	AssertRCReturn(rc, rc);
4756
4757	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4758	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4759	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4760
4761	typedef enum
4762	{
4763	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4764	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4765	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4766	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4767	} VMXREFLECTXCPT;
4768
4769	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4770	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4771	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4772	{
4773	enmReflect = VMXREFLECTXCPT_XCPT;
4774	#ifdef VBOX_STRICT
4775	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4776	&& uExitVector == X86_XCPT_PF)
4777	{
4778	Log(("IDT: Contributory #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4779	}
4780	#endif
4781	if ( uExitVector == X86_XCPT_PF
4782	&& uIdtVector == X86_XCPT_PF)
4783	{
4784	pVmxTransient->fVectoringPF = true;
4785	Log(("IDT: Vectoring #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4786	}
4787	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4788	&& hmR0VmxIsContributoryXcpt(uExitVector)
4789	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4790	\|\| uIdtVector == X86_XCPT_PF))
4791	{
4792	enmReflect = VMXREFLECTXCPT_DF;
4793	}
4794	else if (uIdtVector == X86_XCPT_DF)
4795	enmReflect = VMXREFLECTXCPT_TF;
4796	}
4797	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4798	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4799	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4800	{
4801	/*
4802	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4803	* (whatever they are) as they reoccur when restarting the instruction.
4804	*/
4805	enmReflect = VMXREFLECTXCPT_XCPT;
4806	}
4807
4808	switch (enmReflect)
4809	{
4810	case VMXREFLECTXCPT_XCPT:
4811	{
4812	uint32_t u32ErrCode = 0;
4813	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4814	{
4815	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4816	AssertRCReturn(rc, rc);
4817	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4818	}
4819
4820	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4821	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4822	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4823	rc = VINF_SUCCESS;
4824	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4825	pVCpu->hm.s.Event.u32ErrCode));
4826	break;
4827	}
4828
4829	case VMXREFLECTXCPT_DF:
4830	{
4831	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4832	rc = VINF_VMX_DOUBLE_FAULT;
4833	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4834	uIdtVector, uExitVector));
4835	break;
4836	}
4837
4838	case VMXREFLECTXCPT_TF:
4839	{
4840	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4841	rc = VINF_EM_RESET;
4842	break;
4843	}
4844
4845	default:
4846	Assert(rc == VINF_SUCCESS);
4847	break;
4848	}
4849	}
4850	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4851	return rc;
4852	}
4853
4854
4855	/**
4856	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4857	*
4858	* @returns VBox status code.
4859	* @param pVCpu Pointer to the VMCPU.
4860	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4861	* out-of-sync. Make sure to update the required fields
4862	* before using them.
4863	*
4864	* @remarks No-long-jump zone!!!
4865	*/
4866	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4867	{
4868	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4869	{
4870	uint32_t uVal = 0;
4871	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4872	AssertRCReturn(rc, rc);
4873	uint32_t uShadow = 0;
4874	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4875	AssertRCReturn(rc, rc);
4876
4877	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
4878	CPUMSetGuestCR0(pVCpu, uVal);
4879	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4880	}
4881	return VINF_SUCCESS;
4882	}
4883
4884
4885	/**
4886	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4887	*
4888	* @returns VBox status code.
4889	* @param pVCpu Pointer to the VMCPU.
4890	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4891	* out-of-sync. Make sure to update the required fields
4892	* before using them.
4893	*
4894	* @remarks No-long-jump zone!!!
4895	*/
4896	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4897	{
4898	int rc = VINF_SUCCESS;
4899	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4900	{
4901	uint32_t uVal = 0;
4902	uint32_t uShadow = 0;
4903	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4904	AssertRCReturn(rc, rc);
4905	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4906	AssertRCReturn(rc, rc);
4907
4908	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
4909	CPUMSetGuestCR4(pVCpu, uVal);
4910	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4911	}
4912	return rc;
4913	}
4914
4915
4916	/**
4917	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4918	*
4919	* @returns VBox status code.
4920	* @param pVCpu Pointer to the VMCPU.
4921	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4922	* out-of-sync. Make sure to update the required fields
4923	* before using them.
4924	*
4925	* @remarks No-long-jump zone!!!
4926	*/
4927	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4928	{
4929	int rc = VINF_SUCCESS;
4930	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4931	{
4932	uint64_t u64Val = 0;
4933	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
4934	AssertRCReturn(rc, rc);
4935
4936	pMixedCtx->rip = u64Val;
4937	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4938	}
4939	return rc;
4940	}
4941
4942
4943	/**
4944	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4945	*
4946	* @returns VBox status code.
4947	* @param pVCpu Pointer to the VMCPU.
4948	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4949	* out-of-sync. Make sure to update the required fields
4950	* before using them.
4951	*
4952	* @remarks No-long-jump zone!!!
4953	*/
4954	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4955	{
4956	int rc = VINF_SUCCESS;
4957	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4958	{
4959	uint64_t u64Val = 0;
4960	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
4961	AssertRCReturn(rc, rc);
4962
4963	pMixedCtx->rsp = u64Val;
4964	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4965	}
4966	return rc;
4967	}
4968
4969
4970	/**
4971	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4972	*
4973	* @returns VBox status code.
4974	* @param pVCpu Pointer to the VMCPU.
4975	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4976	* out-of-sync. Make sure to update the required fields
4977	* before using them.
4978	*
4979	* @remarks No-long-jump zone!!!
4980	*/
4981	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4982	{
4983	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4984	{
4985	uint32_t uVal = 0;
4986	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4987	AssertRCReturn(rc, rc);
4988
4989	pMixedCtx->eflags.u32 = uVal;
4990	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
4991	{
4992	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4993	Log(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
4994
4995	pMixedCtx->eflags.Bits.u1VM = 0;
4996	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4997	}
4998
4999	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5000	}
5001	return VINF_SUCCESS;
5002	}
5003
5004
5005	/**
5006	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5007	* guest-CPU context.
5008	*/
5009	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5010	{
5011	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5012	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5013	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5014	return rc;
5015	}
5016
5017
5018	/**
5019	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5020	* from the guest-state area in the VMCS.
5021	*
5022	* @param pVCpu Pointer to the VMCPU.
5023	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5024	* out-of-sync. Make sure to update the required fields
5025	* before using them.
5026	*
5027	* @remarks No-long-jump zone!!!
5028	*/
5029	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5030	{
5031	uint32_t uIntrState = 0;
5032	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5033	AssertRC(rc);
5034
5035	if (!uIntrState)
5036	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5037	else
5038	{
5039	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5040	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5041	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5042	AssertRC(rc);
5043	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5044	AssertRC(rc);
5045
5046	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5047	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5048	}
5049	}
5050
5051
5052	/**
5053	* Saves the guest's activity state.
5054	*
5055	* @returns VBox status code.
5056	* @param pVCpu Pointer to the VMCPU.
5057	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5058	* out-of-sync. Make sure to update the required fields
5059	* before using them.
5060	*
5061	* @remarks No-long-jump zone!!!
5062	*/
5063	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5064	{
5065	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5066	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5067	return VINF_SUCCESS;
5068	}
5069
5070
5071	/**
5072	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5073	* the current VMCS into the guest-CPU context.
5074	*
5075	* @returns VBox status code.
5076	* @param pVCpu Pointer to the VMCPU.
5077	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5078	* out-of-sync. Make sure to update the required fields
5079	* before using them.
5080	*
5081	* @remarks No-long-jump zone!!!
5082	*/
5083	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5084	{
5085	int rc = VINF_SUCCESS;
5086	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5087	{
5088	uint32_t u32Val = 0;
5089	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5090	pMixedCtx->SysEnter.cs = u32Val;
5091	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5092	}
5093
5094	uint64_t u64Val = 0;
5095	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5096	{
5097	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5098	pMixedCtx->SysEnter.eip = u64Val;
5099	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5100	}
5101	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5102	{
5103	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5104	pMixedCtx->SysEnter.esp = u64Val;
5105	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5106	}
5107	return rc;
5108	}
5109
5110
5111	/**
5112	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5113	* context.
5114	*
5115	* @returns VBox status code.
5116	* @param pVCpu Pointer to the VMCPU.
5117	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5118	* out-of-sync. Make sure to update the required fields
5119	* before using them.
5120	*
5121	* @remarks No-long-jump zone!!!
5122	*/
5123	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5124	{
5125	int rc = VINF_SUCCESS;
5126	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5127	{
5128	uint64_t u64Val = 0;
5129	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5130	pMixedCtx->fs.u64Base = u64Val;
5131	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5132	}
5133	return rc;
5134	}
5135
5136
5137	/**
5138	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5139	* context.
5140	*
5141	* @returns VBox status code.
5142	* @param pVCpu Pointer to the VMCPU.
5143	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5144	* out-of-sync. Make sure to update the required fields
5145	* before using them.
5146	*
5147	* @remarks No-long-jump zone!!!
5148	*/
5149	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5150	{
5151	int rc = VINF_SUCCESS;
5152	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5153	{
5154	uint64_t u64Val = 0;
5155	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5156	pMixedCtx->gs.u64Base = u64Val;
5157	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5158	}
5159	return rc;
5160	}
5161
5162
5163	/**
5164	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5165	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5166	* and TSC_AUX.
5167	*
5168	* @returns VBox status code.
5169	* @param pVCpu Pointer to the VMCPU.
5170	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5171	* out-of-sync. Make sure to update the required fields
5172	* before using them.
5173	*
5174	* @remarks No-long-jump zone!!!
5175	*/
5176	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5177	{
5178	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5179	return VINF_SUCCESS;
5180
5181	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5182	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5183	{
5184	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5185	pMsr += i;
5186	switch (pMsr->u32IndexMSR)
5187	{
5188	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5189	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5190	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5191	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5192	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5193	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5194	default:
5195	{
5196	AssertFailed();
5197	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5198	}
5199	}
5200	}
5201	#endif
5202
5203	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5204	return VINF_SUCCESS;
5205	}
5206
5207
5208	/**
5209	* Saves the guest control registers from the current VMCS into the guest-CPU
5210	* context.
5211	*
5212	* @returns VBox status code.
5213	* @param pVCpu Pointer to the VMCPU.
5214	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5215	* out-of-sync. Make sure to update the required fields
5216	* before using them.
5217	*
5218	* @remarks No-long-jump zone!!!
5219	*/
5220	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5221	{
5222	/* Guest CR0. Guest FPU. */
5223	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5224	AssertRCReturn(rc, rc);
5225
5226	/* Guest CR4. */
5227	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5228	AssertRCReturn(rc, rc);
5229
5230	/* Guest CR2 - updated always during the world-switch or in #PF. */
5231	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5232	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5233	{
5234	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5235	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5236
5237	PVM pVM = pVCpu->CTX_SUFF(pVM);
5238	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5239	\|\| ( pVM->hm.s.fNestedPaging
5240	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5241	{
5242	uint64_t u64Val = 0;
5243	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5244	if (pMixedCtx->cr3 != u64Val)
5245	{
5246	CPUMSetGuestCR3(pVCpu, u64Val);
5247	if (VMMRZCallRing3IsEnabled(pVCpu))
5248	{
5249	PGMUpdateCR3(pVCpu, u64Val);
5250	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5251	}
5252	else
5253	{
5254	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5255	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5256	}
5257	}
5258
5259	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5260	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5261	{
5262	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5263	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5264	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5265	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5266
5267	if (VMMRZCallRing3IsEnabled(pVCpu))
5268	{
5269	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5270	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5271	}
5272	else
5273	{
5274	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5275	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5276	}
5277	}
5278	}
5279
5280	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5281	}
5282
5283	/*
5284	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5285	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5286	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5287	*
5288	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5289	*/
5290	if (VMMRZCallRing3IsEnabled(pVCpu))
5291	{
5292	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5293	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5294
5295	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5296	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5297
5298	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5299	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5300	}
5301
5302	return rc;
5303	}
5304
5305
5306	/**
5307	* Reads a guest segment register from the current VMCS into the guest-CPU
5308	* context.
5309	*
5310	* @returns VBox status code.
5311	* @param pVCpu Pointer to the VMCPU.
5312	* @param idxSel Index of the selector in the VMCS.
5313	* @param idxLimit Index of the segment limit in the VMCS.
5314	* @param idxBase Index of the segment base in the VMCS.
5315	* @param idxAccess Index of the access rights of the segment in the VMCS.
5316	* @param pSelReg Pointer to the segment selector.
5317	*
5318	* @remarks No-long-jump zone!!!
5319	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5320	* macro as that takes care of whether to read from the VMCS cache or
5321	* not.
5322	*/
5323	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5324	PCPUMSELREG pSelReg)
5325	{
5326	uint32_t u32Val = 0;
5327	int rc = VMXReadVmcs32(idxSel, &u32Val);
5328	AssertRCReturn(rc, rc);
5329	pSelReg->Sel = (uint16_t)u32Val;
5330	pSelReg->ValidSel = (uint16_t)u32Val;
5331	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5332
5333	rc = VMXReadVmcs32(idxLimit, &u32Val);
5334	AssertRCReturn(rc, rc);
5335	pSelReg->u32Limit = u32Val;
5336
5337	uint64_t u64Val = 0;
5338	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5339	AssertRCReturn(rc, rc);
5340	pSelReg->u64Base = u64Val;
5341
5342	rc = VMXReadVmcs32(idxAccess, &u32Val);
5343	AssertRCReturn(rc, rc);
5344	pSelReg->Attr.u = u32Val;
5345
5346	/*
5347	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5348	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5349	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5350	*
5351	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5352	*/
5353	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5354	{
5355	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5356	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5357	}
5358	return VINF_SUCCESS;
5359	}
5360
5361
5362	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5363	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5364	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5365	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5366	#else
5367	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5368	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5369	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5370	#endif
5371
5372
5373	/**
5374	* Saves the guest segment registers from the current VMCS into the guest-CPU
5375	* context.
5376	*
5377	* @returns VBox status code.
5378	* @param pVCpu Pointer to the VMCPU.
5379	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5380	* out-of-sync. Make sure to update the required fields
5381	* before using them.
5382	*
5383	* @remarks No-long-jump zone!!!
5384	*/
5385	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5386	{
5387	/* Guest segment registers. */
5388	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5389	{
5390	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5391	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5392	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5393	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5394	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5395	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5396	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5397
5398	/* Restore segment attributes for real-on-v86 mode hack. */
5399	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5400	{
5401	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5402	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5403	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5404	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5405	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5406	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5407	}
5408	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5409	}
5410
5411	return VINF_SUCCESS;
5412	}
5413
5414
5415	/**
5416	* Saves the guest descriptor table registers and task register from the current
5417	* VMCS into the guest-CPU context.
5418	*
5419	* @returns VBox status code.
5420	* @param pVCpu Pointer to the VMCPU.
5421	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5422	* out-of-sync. Make sure to update the required fields
5423	* before using them.
5424	*
5425	* @remarks No-long-jump zone!!!
5426	*/
5427	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5428	{
5429	int rc = VINF_SUCCESS;
5430
5431	/* Guest LDTR. */
5432	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5433	{
5434	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5435	AssertRCReturn(rc, rc);
5436	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5437	}
5438
5439	/* Guest GDTR. */
5440	uint64_t u64Val = 0;
5441	uint32_t u32Val = 0;
5442	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5443	{
5444	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5445	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5446	pMixedCtx->gdtr.pGdt = u64Val;
5447	pMixedCtx->gdtr.cbGdt = u32Val;
5448	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5449	}
5450
5451	/* Guest IDTR. */
5452	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5453	{
5454	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5455	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5456	pMixedCtx->idtr.pIdt = u64Val;
5457	pMixedCtx->idtr.cbIdt = u32Val;
5458	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5459	}
5460
5461	/* Guest TR. */
5462	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5463	{
5464	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5465	AssertRCReturn(rc, rc);
5466
5467	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5468	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5469	{
5470	rc = VMXLOCAL_READ_SEG(TR, tr);
5471	AssertRCReturn(rc, rc);
5472	}
5473	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5474	}
5475	return rc;
5476	}
5477
5478	#undef VMXLOCAL_READ_SEG
5479
5480
5481	/**
5482	* Saves the guest debug registers from the current VMCS into the guest-CPU
5483	* context.
5484	*
5485	* @returns VBox status code.
5486	* @param pVCpu Pointer to the VMCPU.
5487	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5488	* out-of-sync. Make sure to update the required fields
5489	* before using them.
5490	*
5491	* @remarks No-long-jump zone!!!
5492	*/
5493	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5494	{
5495	int rc = VINF_SUCCESS;
5496	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5497	{
5498	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5499	uint32_t u32Val;
5500	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5501	pMixedCtx->dr[7] = u32Val;
5502
5503	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5504	}
5505	return rc;
5506	}
5507
5508
5509	/**
5510	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5511	*
5512	* @returns VBox status code.
5513	* @param pVCpu Pointer to the VMCPU.
5514	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5515	* out-of-sync. Make sure to update the required fields
5516	* before using them.
5517	*
5518	* @remarks No-long-jump zone!!!
5519	*/
5520	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5521	{
5522	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5523	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5524	return VINF_SUCCESS;
5525	}
5526
5527
5528	/**
5529	* Saves the entire guest state from the currently active VMCS into the
5530	* guest-CPU context. This essentially VMREADs all guest-data.
5531	*
5532	* @returns VBox status code.
5533	* @param pVCpu Pointer to the VMCPU.
5534	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5535	* out-of-sync. Make sure to update the required fields
5536	* before using them.
5537	*/
5538	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5539	{
5540	Assert(pVCpu);
5541	Assert(pMixedCtx);
5542
5543	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5544	return VINF_SUCCESS;
5545
5546	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5547	there is no real need to. */
5548	if (VMMRZCallRing3IsEnabled(pVCpu))
5549	VMMR0LogFlushDisable(pVCpu);
5550	else
5551	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5552	LogFunc(("\n"));
5553
5554	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5555	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5556
5557	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5558	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5559
5560	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5561	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5562
5563	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5564	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5565
5566	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5567	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5568
5569	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5570	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5571
5572	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5573	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5574
5575	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5576	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5577
5578	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5579	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5580
5581	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5582	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5583
5584	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5585	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5586
5587	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5588	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5589
5590	if (VMMRZCallRing3IsEnabled(pVCpu))
5591	VMMR0LogFlushEnable(pVCpu);
5592
5593	return rc;
5594	}
5595
5596
5597	/**
5598	* Check per-VM and per-VCPU force flag actions that require us to go back to
5599	* ring-3 for one reason or another.
5600	*
5601	* @returns VBox status code (information status code included).
5602	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5603	* ring-3.
5604	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5605	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5606	* interrupts)
5607	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5608	* all EMTs to be in ring-3.
5609	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5610	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5611	* to the EM loop.
5612	*
5613	* @param pVM Pointer to the VM.
5614	* @param pVCpu Pointer to the VMCPU.
5615	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5616	* out-of-sync. Make sure to update the required fields
5617	* before using them.
5618	*/
5619	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5620	{
5621	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5622
5623	int rc = VERR_INTERNAL_ERROR_5;
5624	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5625	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5626	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5627	{
5628	/* We need the control registers now, make sure the guest-CPU context is updated. */
5629	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5630	AssertRCReturn(rc, rc);
5631
5632	/* Pending HM CR3 sync. */
5633	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5634	{
5635	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5636	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5637	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5638	}
5639
5640	/* Pending HM PAE PDPEs. */
5641	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5642	{
5643	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5644	AssertRC(rc);
5645	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5646	}
5647
5648	/* Pending PGM C3 sync. */
5649	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5650	{
5651	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5652	if (rc != VINF_SUCCESS)
5653	{
5654	AssertRC(rc);
5655	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5656	return rc;
5657	}
5658	}
5659
5660	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5661	/* -XXX- what was that about single stepping? */
5662	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5663	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5664	{
5665	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5666	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5667	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5668	return rc;
5669	}
5670
5671	/* Pending VM request packets, such as hardware interrupts. */
5672	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5673	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5674	{
5675	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5676	return VINF_EM_PENDING_REQUEST;
5677	}
5678
5679	/* Pending PGM pool flushes. */
5680	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5681	{
5682	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5683	return VINF_PGM_POOL_FLUSH_PENDING;
5684	}
5685
5686	/* Pending DMA requests. */
5687	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5688	{
5689	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5690	return VINF_EM_RAW_TO_R3;
5691	}
5692	}
5693
5694	/* Paranoia. */
5695	Assert(rc != VERR_EM_INTERPRETER);
5696	return VINF_SUCCESS;
5697	}
5698
5699
5700	/**
5701	* Converts any TRPM trap into a pending VMX event. This is typically used when
5702	* entering from ring-3 (not longjmp returns).
5703	*
5704	* @param pVCpu Pointer to the VMCPU.
5705	*/
5706	static void hmR0VmxTRPMTrapToPendingEvent(PVMCPU pVCpu)
5707	{
5708	Assert(TRPMHasTrap(pVCpu));
5709	Assert(!pVCpu->hm.s.Event.fPending);
5710
5711	uint8_t uVector;
5712	TRPMEVENT enmTrpmEvent;
5713	RTGCUINT uErrCode;
5714	RTGCUINTPTR GCPtrFaultAddress;
5715	uint8_t cbInstr;
5716
5717	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5718	AssertRC(rc);
5719
5720	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5721	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5722	if (enmTrpmEvent == TRPM_TRAP)
5723	{
5724	switch (uVector)
5725	{
5726	case X86_XCPT_BP:
5727	case X86_XCPT_OF:
5728	{
5729	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5730	break;
5731	}
5732
5733	case X86_XCPT_PF:
5734	case X86_XCPT_DF:
5735	case X86_XCPT_TS:
5736	case X86_XCPT_NP:
5737	case X86_XCPT_SS:
5738	case X86_XCPT_GP:
5739	case X86_XCPT_AC:
5740	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5741	/* no break! */
5742	default:
5743	{
5744	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5745	break;
5746	}
5747	}
5748	}
5749	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5750	{
5751	if (uVector != X86_XCPT_NMI)
5752	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5753	else
5754	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5755	}
5756	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5757	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5758	else
5759	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5760
5761	rc = TRPMResetTrap(pVCpu);
5762	AssertRC(rc);
5763	Log(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5764	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5765	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5766	}
5767
5768
5769	/**
5770	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5771	* VT-x to execute any instruction.
5772	*
5773	* @param pvCpu Pointer to the VMCPU.
5774	*/
5775	static void hmR0VmxPendingEventToTRPMTrap(PVMCPU pVCpu)
5776	{
5777	Assert(pVCpu->hm.s.Event.fPending);
5778
5779	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5780	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5781	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5782	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5783
5784	/* If a trap was already pending, we did something wrong! */
5785	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5786
5787	TRPMEVENT enmTrapType;
5788	switch (uVectorType)
5789	{
5790	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5791	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5792	enmTrapType = TRPM_HARDWARE_INT;
5793	break;
5794	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5795	enmTrapType = TRPM_SOFTWARE_INT;
5796	break;
5797	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5798	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5799	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5800	enmTrapType = TRPM_TRAP;
5801	break;
5802	default:
5803	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5804	enmTrapType = TRPM_32BIT_HACK;
5805	break;
5806	}
5807
5808	Log(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5809
5810	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5811	AssertRC(rc);
5812
5813	if (fErrorCodeValid)
5814	TRPMSetErrorCode(pVCpu, uErrorCode);
5815	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5816	&& uVector == X86_XCPT_PF)
5817	{
5818	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5819	}
5820	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5821	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5822	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5823	{
5824	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5825	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5826	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5827	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5828	}
5829	pVCpu->hm.s.Event.fPending = false;
5830	}
5831
5832
5833	/**
5834	* Does the necessary state syncing before doing a longjmp to ring-3.
5835	*
5836	* @param pVM Pointer to the VM.
5837	* @param pVCpu Pointer to the VMCPU.
5838	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5839	* out-of-sync. Make sure to update the required fields
5840	* before using them.
5841	* @param rcExit The reason for exiting to ring-3. Can be
5842	* VINF_VMM_UNKNOWN_RING3_CALL.
5843	*
5844	* @remarks No-long-jmp zone!!!
5845	*/
5846	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5847	{
5848	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5849	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5850
5851	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5852	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5853	AssertRC(rc);
5854
5855	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5856	if (CPUMIsGuestFPUStateActive(pVCpu))
5857	{
5858	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5859	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5860	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5861	}
5862
5863	/* Restore debug registers if necessary and resync on next R0 reentry. */
5864	if (CPUMIsGuestDebugStateActive(pVCpu))
5865	{
5866	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5867	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5868	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5869	}
5870	else if (CPUMIsHyperDebugStateActive(pVCpu))
5871	{
5872	CPUMR0LoadHostDebugState(pVM, pVCpu);
5873	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5874	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5875	}
5876
5877	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
5878	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
5879	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
5880	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5881	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5882	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5883	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5884	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5885	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5886	}
5887
5888
5889	/**
5890	* An action requires us to go back to ring-3. This function does the necessary
5891	* steps before we can safely return to ring-3. This is not the same as longjmps
5892	* to ring-3, this is voluntary.
5893	*
5894	* @param pVM Pointer to the VM.
5895	* @param pVCpu Pointer to the VMCPU.
5896	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5897	* out-of-sync. Make sure to update the required fields
5898	* before using them.
5899	* @param rcExit The reason for exiting to ring-3. Can be
5900	* VINF_VMM_UNKNOWN_RING3_CALL.
5901	*/
5902	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5903	{
5904	Assert(pVM);
5905	Assert(pVCpu);
5906	Assert(pMixedCtx);
5907	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5908
5909	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5910	{
5911	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5912	return;
5913	}
5914	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5915	{
5916	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5917	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5918	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5919	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5920	return;
5921	}
5922
5923	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5924	VMMRZCallRing3Disable(pVCpu);
5925	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5926
5927	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5928	if (pVCpu->hm.s.Event.fPending)
5929	{
5930	hmR0VmxPendingEventToTRPMTrap(pVCpu);
5931	Assert(!pVCpu->hm.s.Event.fPending);
5932	}
5933
5934	/* Sync. the guest state. */
5935	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5936	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5937
5938	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5939	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5940	\| CPUM_CHANGED_LDTR
5941	\| CPUM_CHANGED_GDTR
5942	\| CPUM_CHANGED_IDTR
5943	\| CPUM_CHANGED_TR
5944	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5945
5946	/* On our way back from ring-3 the following needs to be done. */
5947	/** @todo This can change with preemption hooks. */
5948	if (rcExit == VINF_EM_RAW_INTERRUPT)
5949	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5950	else
5951	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5952
5953	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5954	VMMRZCallRing3Enable(pVCpu);
5955	}
5956
5957
5958	/**
5959	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5960	* longjump to ring-3 and possibly get preempted.
5961	*
5962	* @param pVCpu Pointer to the VMCPU.
5963	* @param enmOperation The operation causing the ring-3 longjump.
5964	* @param pvUser The user argument (pointer to the possibly
5965	* out-of-date guest-CPU context).
5966	*
5967	* @remarks Must never be called with @a enmOperation ==
5968	* VMMCALLRING3_VM_R0_ASSERTION.
5969	*/
5970	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5971	{
5972	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5973	Assert(pVCpu);
5974	Assert(pvUser);
5975	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5976	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5977
5978	VMMRZCallRing3Disable(pVCpu);
5979	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5980	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5981	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5982	VMMRZCallRing3Enable(pVCpu);
5983	}
5984
5985
5986	/**
5987	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5988	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5989	*
5990	* @returns VBox status code.
5991	* @param pVCpu Pointer to the VMCPU.
5992	*/
5993	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5994	{
5995	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
5996	{
5997	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
5998	{
5999	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6000	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6001	AssertRC(rc);
6002	}
6003	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6004	}
6005
6006
6007	/**
6008	* Injects any pending events into the guest if the guest is in a state to
6009	* receive them.
6010	*
6011	* @returns VBox status code (informational status codes included).
6012	* @param pVCpu Pointer to the VMCPU.
6013	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6014	* out-of-sync. Make sure to update the required fields
6015	* before using them.
6016	*/
6017	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6018	{
6019	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6020	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6021	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6022	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6023
6024	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6025	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6026	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6027	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6028	Assert(!TRPMHasTrap(pVCpu));
6029
6030	int rc = VINF_SUCCESS;
6031	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6032	{
6033	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6034	bool fInject = true;
6035	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6036	{
6037	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6038	AssertRCReturn(rc, rc);
6039	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6040	if ( fBlockInt
6041	\|\| fBlockSti
6042	\|\| fBlockMovSS)
6043	{
6044	fInject = false;
6045	}
6046	}
6047	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6048	&& ( fBlockMovSS
6049	\|\| fBlockSti))
6050	{
6051	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6052	fInject = false;
6053	}
6054
6055	if (fInject)
6056	{
6057	Log(("Injecting pending event\n"));
6058	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6059	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6060	AssertRCReturn(rc, rc);
6061	pVCpu->hm.s.Event.fPending = false;
6062	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
6063	}
6064	else
6065	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6066	} /** @todo SMI. SMIs take priority over NMIs. */
6067	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6068	{
6069	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6070	if ( !fBlockMovSS
6071	&& !fBlockSti)
6072	{
6073	Log(("Injecting NMI\n"));
6074	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6075	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6076	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6077	0 /* GCPtrFaultAddress */, &uIntrState);
6078	AssertRCReturn(rc, rc);
6079	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6080	}
6081	else
6082	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6083	}
6084	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
6085	{
6086	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6087	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6088	AssertRCReturn(rc, rc);
6089	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6090	if ( !fBlockInt
6091	&& !fBlockSti
6092	&& !fBlockMovSS)
6093	{
6094	uint8_t u8Interrupt;
6095	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6096	if (RT_SUCCESS(rc))
6097	{
6098	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6099	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6100	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6101	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6102	0 /* GCPtrFaultAddress */, &uIntrState);
6103	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6104	}
6105	else
6106	{
6107	/** @todo Does this actually happen? If not turn it into an assertion. */
6108	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6109	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6110	rc = VINF_SUCCESS;
6111	}
6112	}
6113	else
6114	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6115	}
6116
6117	/*
6118	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6119	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6120	*/
6121	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6122	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6123	int rc2 = VINF_SUCCESS;
6124	if ( fBlockSti
6125	\|\| fBlockMovSS)
6126	{
6127	if (!DBGFIsStepping(pVCpu))
6128	{
6129	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6130	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6131	{
6132	/*
6133	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6134	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6135	*/
6136	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6137	AssertRCReturn(rc, rc);
6138	}
6139	}
6140	else
6141	{
6142	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6143	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6144	uIntrState = 0;
6145	}
6146	}
6147
6148	/*
6149	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6150	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6151	*/
6152	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6153	AssertRC(rc2);
6154
6155	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6156	return rc;
6157	}
6158
6159
6160	/**
6161	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6162	*
6163	* @param pVCpu Pointer to the VMCPU.
6164	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6165	* out-of-sync. Make sure to update the required fields
6166	* before using them.
6167	*/
6168	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6169	{
6170	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6171	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6172	}
6173
6174
6175	/**
6176	* Injects a double-fault (#DF) exception into the VM.
6177	*
6178	* @returns VBox status code (informational status code included).
6179	* @param pVCpu Pointer to the VMCPU.
6180	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6181	* out-of-sync. Make sure to update the required fields
6182	* before using them.
6183	*/
6184	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6185	{
6186	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6187	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6188	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6189	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6190	puIntrState);
6191	}
6192
6193
6194	/**
6195	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6196	*
6197	* @param pVCpu Pointer to the VMCPU.
6198	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6199	* out-of-sync. Make sure to update the required fields
6200	* before using them.
6201	*/
6202	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6203	{
6204	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6205	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6206	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6207	}
6208
6209
6210	/**
6211	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6212	*
6213	* @param pVCpu Pointer to the VMCPU.
6214	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6215	* out-of-sync. Make sure to update the required fields
6216	* before using them.
6217	* @param cbInstr The value of RIP that is to be pushed on the guest
6218	* stack.
6219	*/
6220	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6221	{
6222	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6223	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6224	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6225	}
6226
6227
6228	/**
6229	* Injects a general-protection (#GP) fault into the VM.
6230	*
6231	* @returns VBox status code (informational status code included).
6232	* @param pVCpu Pointer to the VMCPU.
6233	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6234	* out-of-sync. Make sure to update the required fields
6235	* before using them.
6236	* @param u32ErrorCode The error code associated with the #GP.
6237	*/
6238	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6239	uint32_t *puIntrState)
6240	{
6241	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6242	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6243	if (fErrorCodeValid)
6244	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6245	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6246	puIntrState);
6247	}
6248
6249
6250	/**
6251	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6252	*
6253	* @param pVCpu Pointer to the VMCPU.
6254	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6255	* out-of-sync. Make sure to update the required fields
6256	* before using them.
6257	* @param uVector The software interrupt vector number.
6258	* @param cbInstr The value of RIP that is to be pushed on the guest
6259	* stack.
6260	*/
6261	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6262	{
6263	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6264	if ( uVector == X86_XCPT_BP
6265	\|\| uVector == X86_XCPT_OF)
6266	{
6267	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6268	}
6269	else
6270	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6271	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6272	}
6273
6274
6275	/**
6276	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6277	* stack.
6278	*
6279	* @returns VBox status code (information status code included).
6280	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6281	* @param pVM Pointer to the VM.
6282	* @param pMixedCtx Pointer to the guest-CPU context.
6283	* @param uValue The value to push to the guest stack.
6284	*/
6285	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6286	{
6287	/*
6288	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6289	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6290	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6291	*/
6292	if (pMixedCtx->sp == 1)
6293	return VINF_EM_RESET;
6294	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6295	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6296	AssertRCReturn(rc, rc);
6297	return rc;
6298	}
6299
6300
6301	/**
6302	* Injects an event into the guest upon VM-entry by updating the relevant fields
6303	* in the VM-entry area in the VMCS.
6304	*
6305	* @returns VBox status code (informational error codes included).
6306	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6307	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6308	*
6309	* @param pVCpu Pointer to the VMCPU.
6310	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6311	* be out-of-sync. Make sure to update the required
6312	* fields before using them.
6313	* @param u64IntrInfo The VM-entry interruption-information field.
6314	* @param cbInstr The VM-entry instruction length in bytes (for
6315	* software interrupts, exceptions and privileged
6316	* software exceptions).
6317	* @param u32ErrCode The VM-entry exception error code.
6318	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6319	* @param puIntrState Pointer to the current guest interruptibility-state.
6320	* This interruptibility-state will be updated if
6321	* necessary. This cannot not be NULL.
6322	*
6323	* @remarks No-long-jump zone!!!
6324	* @remarks Requires CR0!
6325	*/
6326	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6327	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6328	{
6329	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6330	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6331	Assert(puIntrState);
6332	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6333
6334	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6335	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6336
6337	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6338	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6339	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6340
6341	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6342
6343	/* We require CR0 to check if the guest is in real-mode. */
6344	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6345	AssertRCReturn(rc, rc);
6346
6347	/*
6348	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6349	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6350	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6351	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6352	*/
6353	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6354	{
6355	PVM pVM = pVCpu->CTX_SUFF(pVM);
6356	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6357	{
6358	Assert(PDMVmmDevHeapIsEnabled(pVM));
6359	Assert(pVM->hm.s.vmx.pRealModeTSS);
6360
6361	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6362	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6363	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6364	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6365	AssertRCReturn(rc, rc);
6366	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6367
6368	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6369	const size_t cbIdtEntry = 4;
6370	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6371	{
6372	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6373	if (uVector == X86_XCPT_DF)
6374	return VINF_EM_RESET;
6375	else if (uVector == X86_XCPT_GP)
6376	{
6377	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6378	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6379	}
6380
6381	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6382	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6383	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6384	}
6385
6386	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6387	uint16_t uGuestIp = pMixedCtx->ip;
6388	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6389	{
6390	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6391	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6392	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6393	}
6394	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6395	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6396
6397	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6398	uint16_t offIdtEntry = 0;
6399	RTSEL selIdtEntry = 0;
6400	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6401	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6402	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6403	AssertRCReturn(rc, rc);
6404
6405	/* Construct the stack frame for the interrupt/exception handler. */
6406	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6407	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6408	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6409	AssertRCReturn(rc, rc);
6410
6411	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6412	if (rc == VINF_SUCCESS)
6413	{
6414	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6415	pMixedCtx->rip = offIdtEntry;
6416	pMixedCtx->cs.Sel = selIdtEntry;
6417	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6418	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6419	&& uVector == X86_XCPT_PF)
6420	{
6421	pMixedCtx->cr2 = GCPtrFaultAddress;
6422	}
6423	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6424	\| HM_CHANGED_GUEST_RIP
6425	\| HM_CHANGED_GUEST_RFLAGS
6426	\| HM_CHANGED_GUEST_RSP;
6427
6428	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6429	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6430	{
6431	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6432	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6433	Log(("Clearing inhibition due to STI.\n"));
6434	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6435	}
6436	Log(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6437	}
6438	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6439	return rc;
6440	}
6441	else
6442	{
6443	/*
6444	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6445	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6446	*/
6447	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6448	}
6449	}
6450
6451	/* Validate. */
6452	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6453	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6454	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6455
6456	/* Inject. */
6457	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6458	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6459	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6460	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6461
6462	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6463	&& uVector == X86_XCPT_PF)
6464	{
6465	pMixedCtx->cr2 = GCPtrFaultAddress;
6466	}
6467	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RX64\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6468
6469	AssertRCReturn(rc, rc);
6470	return rc;
6471	}
6472
6473
6474	/**
6475	* Enters the VT-x session.
6476	*
6477	* @returns VBox status code.
6478	* @param pVM Pointer to the VM.
6479	* @param pVCpu Pointer to the VMCPU.
6480	* @param pCpu Pointer to the CPU info struct.
6481	*/
6482	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6483	{
6484	AssertPtr(pVM);
6485	AssertPtr(pVCpu);
6486	Assert(pVM->hm.s.vmx.fSupported);
6487	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6488	NOREF(pCpu);
6489
6490	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6491
6492	/* Make sure we're in VMX root mode. */
6493	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6494	if (!(u32HostCR4 & X86_CR4_VMXE))
6495	{
6496	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6497	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6498	}
6499
6500	/* Load the active VMCS as the current one. */
6501	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6502	if (RT_FAILURE(rc))
6503	return rc;
6504
6505	/** @todo this will change with preemption hooks where can can VMRESUME as long
6506	* as we're no preempted. */
6507	pVCpu->hm.s.fResumeVM = false;
6508	return VINF_SUCCESS;
6509	}
6510
6511
6512	/**
6513	* Leaves the VT-x session.
6514	*
6515	* @returns VBox status code.
6516	* @param pVM Pointer to the VM.
6517	* @param pVCpu Pointer to the VMCPU.
6518	* @param pCtx Pointer to the guest-CPU context.
6519	*/
6520	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6521	{
6522	AssertPtr(pVCpu);
6523	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6524	NOREF(pVM);
6525	NOREF(pCtx);
6526
6527	/** @todo this will change with preemption hooks where we only VMCLEAR when
6528	* we are actually going to be preempted, not all the time like we
6529	* currently do. */
6530
6531	/* Restore host-state bits that VT-x only restores partially. */
6532	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6533	{
6534	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6535	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6536	}
6537
6538	/*
6539	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6540	* and mark the VMCS launch-state as "clear".
6541	*/
6542	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6543	return rc;
6544	}
6545
6546
6547	/**
6548	* Saves the host state in the VMCS host-state.
6549	* Sets up the VM-exit MSR-load area.
6550	*
6551	* The CPU state will be loaded from these fields on every successful VM-exit.
6552	*
6553	* @returns VBox status code.
6554	* @param pVM Pointer to the VM.
6555	* @param pVCpu Pointer to the VMCPU.
6556	*
6557	* @remarks No-long-jump zone!!!
6558	*/
6559	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6560	{
6561	AssertPtr(pVM);
6562	AssertPtr(pVCpu);
6563	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6564
6565	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6566
6567	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6568	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6569	return VINF_SUCCESS;
6570
6571	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6572	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6573
6574	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6575	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6576
6577	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6578	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6579
6580	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6581	return rc;
6582	}
6583
6584
6585	/**
6586	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6587	* loaded from these fields on every successful VM-entry.
6588	*
6589	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6590	* Sets up the VM-entry controls.
6591	* Sets up the appropriate VMX non-root function to execute guest code based on
6592	* the guest CPU mode.
6593	*
6594	* @returns VBox status code.
6595	* @param pVM Pointer to the VM.
6596	* @param pVCpu Pointer to the VMCPU.
6597	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6598	* out-of-sync. Make sure to update the required fields
6599	* before using them.
6600	*
6601	* @remarks No-long-jump zone!!!
6602	*/
6603	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6604	{
6605	AssertPtr(pVM);
6606	AssertPtr(pVCpu);
6607	AssertPtr(pMixedCtx);
6608	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6609
6610	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6611
6612	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6613
6614	/* Determine real-on-v86 mode. */
6615	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6616	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6617	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6618	{
6619	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6620	}
6621
6622	/*
6623	* Load the guest-state into the VMCS.
6624	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6625	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6626	*/
6627	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6628	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6629
6630	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6631	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6632
6633	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6634	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6635
6636	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6637	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6638
6639	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6640	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6641	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6642
6643	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6644	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6645
6646	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6647	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6648
6649	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6650	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6651
6652	/* Must be done after hmR0VmxLoadGuestDebugRegs() as it may update eflags.TF for debugging purposes. */
6653	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6654	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6655
6656	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6657	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6658
6659	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6660	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6661	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6662
6663	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6664	return rc;
6665	}
6666
6667
6668	/**
6669	* Does the preparations before executing guest code in VT-x.
6670	*
6671	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6672	* recompiler. We must be cautious what we do here regarding committing
6673	* guest-state information into the the VMCS assuming we assuredly execute the
6674	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6675	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6676	* that the recompiler can (and should) use them when it resumes guest
6677	* execution. Otherwise such operations must be done when we can no longer
6678	* exit to ring-3.
6679	*
6680	* @returns VBox status code (informational status codes included).
6681	* @retval VINF_SUCCESS if we can proceed with running the guest.
6682	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6683	* into the guest.
6684	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6685	*
6686	* @param pVM Pointer to the VM.
6687	* @param pVCpu Pointer to the VMCPU.
6688	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6689	* out-of-sync. Make sure to update the required fields
6690	* before using them.
6691	* @param pVmxTransient Pointer to the VMX transient structure.
6692	*
6693	* @remarks Called with preemption disabled.
6694	*/
6695	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6696	{
6697	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6698
6699	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6700	PGMRZDynMapFlushAutoSet(pVCpu);
6701	#endif
6702
6703	/* Check force flag actions that might require us to go back to ring-3. */
6704	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6705	if (rc != VINF_SUCCESS)
6706	return rc;
6707
6708	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6709	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6710	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6711	{
6712	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6713	RTGCPHYS GCPhysApicBase;
6714	GCPhysApicBase = pMixedCtx->msrApicBase;
6715	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6716
6717	/* Unalias any existing mapping. */
6718	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6719	AssertRCReturn(rc, rc);
6720
6721	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6722	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6723	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6724	AssertRCReturn(rc, rc);
6725
6726	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6727	}
6728
6729	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6730	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6731	pVmxTransient->uEFlags = ASMIntDisableFlags();
6732	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6733	{
6734	ASMSetFlags(pVmxTransient->uEFlags);
6735	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6736	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6737	return VINF_EM_RAW_INTERRUPT;
6738	}
6739	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6740	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6741	#endif
6742
6743	/*
6744	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6745	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6746	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6747	*/
6748	/** @todo Rework event evaluation and injection to be completely separate. */
6749	if (TRPMHasTrap(pVCpu))
6750	hmR0VmxTRPMTrapToPendingEvent(pVCpu);
6751
6752	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6753	AssertRCReturn(rc, rc);
6754	return rc;
6755	}
6756
6757
6758	/**
6759	* Prepares to run guest code in VT-x and we've committed to doing so. This
6760	* means there is no backing out to ring-3 or anywhere else at this
6761	* point.
6762	*
6763	* @param pVM Pointer to the VM.
6764	* @param pVCpu Pointer to the VMCPU.
6765	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6766	* out-of-sync. Make sure to update the required fields
6767	* before using them.
6768	* @param pVmxTransient Pointer to the VMX transient structure.
6769	*
6770	* @remarks Called with preemption disabled.
6771	* @remarks No-long-jump zone!!!
6772	*/
6773	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6774	{
6775	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6776	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6777
6778	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6779	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6780	pVmxTransient->uEFlags = ASMIntDisableFlags();
6781	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6782	#endif
6783
6784	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6785	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6786	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6787	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6788	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6789	#endif
6790	int rc = VINF_SUCCESS;
6791	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6792	{
6793	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6794	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6795	}
6796	else if (pVCpu->hm.s.fContextUseFlags)
6797	{
6798	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6799	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6800	}
6801	AssertRC(rc);
6802	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6803
6804	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6805	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6806	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6807
6808	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6809	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6810	{
6811	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6812	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6813	}
6814
6815	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6816	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6817	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6818
6819	/*
6820	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6821	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6822	*/
6823	if (pVM->hm.s.fTPRPatchingActive)
6824	{
6825	Assert(!CPUMIsGuestInLongMode(pVCpu));
6826
6827	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6828	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6829	AssertRC(rc);
6830
6831	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6832	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6833	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6834	see hmR0VmxLoadGuestApicState(). */
6835	}
6836
6837	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6838	/*
6839	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6840	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6841	*/
6842	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6843	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6844	{
6845	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6846	uint64_t u64HostTscAux = 0;
6847	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6848	AssertRC(rc2);
6849	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6850	}
6851	#endif
6852
6853	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6854	to start executing. */
6855	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6856	}
6857
6858
6859	/**
6860	* Performs some essential restoration of state after running guest code in
6861	* VT-x.
6862	*
6863	* @param pVM Pointer to the VM.
6864	* @param pVCpu Pointer to the VMCPU.
6865	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6866	* out-of-sync. Make sure to update the required fields
6867	* before using them.
6868	* @param pVmxTransient Pointer to the VMX transient structure.
6869	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6870	*
6871	* @remarks Called with interrupts disabled.
6872	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6873	* unconditionally when it is safe to do so.
6874	*/
6875	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6876	{
6877	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6878	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6879
6880	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6881	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6882	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6883	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6884	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6885
6886	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6887	{
6888	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6889	/* Restore host's TSC_AUX. */
6890	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6891	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6892	#endif
6893	/** @todo Find a way to fix hardcoding a guestimate. */
6894	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6895	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6896	}
6897
6898	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6899	Assert(!(ASMGetFlags() & X86_EFL_IF));
6900	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6901
6902	/* Restore the effects of TPR patching if any. */
6903	if (pVM->hm.s.fTPRPatchingActive)
6904	{
6905	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6906	AssertRC(rc);
6907	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6908	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6909	}
6910
6911	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6912	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6913
6914	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6915	uint32_t uExitReason;
6916	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6917	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6918	AssertRC(rc);
6919	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6920	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6921
6922	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6923	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6924
6925	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6926	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6927	{
6928	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6929	return;
6930	}
6931
6932	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6933	{
6934	/* Update the guest interruptibility-state from the VMCS. */
6935	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6936	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6937	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6938	AssertRC(rc);
6939	#endif
6940	/*
6941	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6942	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
6943	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
6944	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
6945	*/
6946	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6947	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6948	{
6949	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6950	AssertRC(rc);
6951	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6952	}
6953	}
6954	}
6955
6956
6957	/**
6958	* Runs the guest code using VT-x.
6959	*
6960	* @returns VBox status code.
6961	* @param pVM Pointer to the VM.
6962	* @param pVCpu Pointer to the VMCPU.
6963	* @param pCtx Pointer to the guest-CPU context.
6964	*
6965	* @remarks Called with preemption disabled.
6966	*/
6967	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6968	{
6969	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6970	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6971
6972	VMXTRANSIENT VmxTransient;
6973	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6974	int rc = VERR_INTERNAL_ERROR_5;
6975	uint32_t cLoops = 0;
6976
6977	for (;; cLoops++)
6978	{
6979	Assert(!HMR0SuspendPending());
6980	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6981	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6982	(unsigned)RTMpCpuId(), cLoops));
6983
6984	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6985	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6986	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6987	if (rc != VINF_SUCCESS)
6988	break;
6989
6990	/*
6991	* No longjmps to ring-3 from this point on!!!
6992	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6993	* This also disables flushing of the R0-logger instance (if any).
6994	*/
6995	VMMRZCallRing3Disable(pVCpu);
6996	VMMRZCallRing3RemoveNotification(pVCpu);
6997	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6998
6999	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7000	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7001
7002	/*
7003	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7004	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7005	*/
7006	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7007	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7008	{
7009	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7010	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7011	return rc;
7012	}
7013
7014	/* Handle the VM-exit. */
7015	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7016	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7017	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7018	HMVMX_START_EXIT_DISPATCH_PROF();
7019	#ifdef HMVMX_USE_FUNCTION_TABLE
7020	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7021	#else
7022	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7023	#endif
7024	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7025	if (rc != VINF_SUCCESS)
7026	break;
7027	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7028	{
7029	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7030	rc = VINF_EM_RAW_INTERRUPT;
7031	break;
7032	}
7033	}
7034
7035	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7036	if (rc == VERR_EM_INTERPRETER)
7037	rc = VINF_EM_RAW_EMULATE_INSTR;
7038	else if (rc == VINF_EM_RESET)
7039	rc = VINF_EM_TRIPLE_FAULT;
7040	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7041	return rc;
7042	}
7043
7044
7045	#ifndef HMVMX_USE_FUNCTION_TABLE
7046	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7047	{
7048	int rc;
7049	switch (rcReason)
7050	{
7051	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7052	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7053	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7054	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7055	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7056	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7057	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7058	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7059	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7060	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7061	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7062	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7063	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7064	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7065	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7066	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7067	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7068	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7069	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7070	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7071	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7072	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7073	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7074	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7075	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7076	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7077	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7078	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7079	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7080	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7081	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7082	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7083	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7084
7085	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7086	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7087	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7088	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7089	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7090	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7091	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7092	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7093	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7094
7095	case VMX_EXIT_VMCALL:
7096	case VMX_EXIT_VMCLEAR:
7097	case VMX_EXIT_VMLAUNCH:
7098	case VMX_EXIT_VMPTRLD:
7099	case VMX_EXIT_VMPTRST:
7100	case VMX_EXIT_VMREAD:
7101	case VMX_EXIT_VMRESUME:
7102	case VMX_EXIT_VMWRITE:
7103	case VMX_EXIT_VMXOFF:
7104	case VMX_EXIT_VMXON:
7105	case VMX_EXIT_INVEPT:
7106	case VMX_EXIT_INVVPID:
7107	case VMX_EXIT_VMFUNC:
7108	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7109	break;
7110	default:
7111	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7112	break;
7113	}
7114	return rc;
7115	}
7116	#endif
7117
7118	#ifdef DEBUG
7119	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7120	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
7121	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7122	# define VMX_ASSERT_PREEMPT_CPUID() \
7123	do \
7124	{ \
7125	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7126	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7127	} while (0)
7128
7129	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7130	do { \
7131	AssertPtr(pVCpu); \
7132	AssertPtr(pMixedCtx); \
7133	AssertPtr(pVmxTransient); \
7134	Assert(pVmxTransient->fVMEntryFailed == false); \
7135	Assert(ASMIntAreEnabled()); \
7136	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7137	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
7138	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
7139	(unsigned)pVCpu->idCpu)); \
7140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7141	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7142	VMX_ASSERT_PREEMPT_CPUID(); \
7143	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7144	} while (0)
7145	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7146	do { \
7147	LogFunc(("\n")); \
7148	} while(0)
7149	#else /* Release builds */
7150	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7151	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7152	#endif
7153
7154
7155	/**
7156	* Advances the guest RIP after reading it from the VMCS.
7157	*
7158	* @returns VBox status code.
7159	* @param pVCpu Pointer to the VMCPU.
7160	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7161	* out-of-sync. Make sure to update the required fields
7162	* before using them.
7163	* @param pVmxTransient Pointer to the VMX transient structure.
7164	*
7165	* @remarks No-long-jump zone!!!
7166	*/
7167	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7168	{
7169	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7170	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7171	AssertRCReturn(rc, rc);
7172
7173	pMixedCtx->rip += pVmxTransient->cbInstr;
7174	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7175	return rc;
7176	}
7177
7178
7179	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7180	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7181	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7182	/**
7183	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7184	*/
7185	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7186	{
7187	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7188	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7189	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
7190	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
7191	Assert(ASMIntAreEnabled());
7192	return VINF_SUCCESS;
7193	#else
7194	return VINF_EM_RAW_INTERRUPT;
7195	#endif
7196	}
7197
7198
7199	/**
7200	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
7201	*/
7202	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7203	{
7204	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7205	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7206
7207	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7208	AssertRCReturn(rc, rc);
7209
7210	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7211	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7212	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7213
7214	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7215	{
7216	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7217	return VINF_EM_RAW_INTERRUPT;
7218	}
7219
7220	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7221	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7222	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7223	{
7224	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7225	return VINF_SUCCESS;
7226	}
7227	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7228	{
7229	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7230	return rc;
7231	}
7232
7233	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7234	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7235	switch (uIntrType)
7236	{
7237	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7238	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7239	/* no break */
7240	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7241	{
7242	switch (uVector)
7243	{
7244	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7245	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7246	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7247	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7248	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7249	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7250	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7251	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7252	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7253	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7254	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7255	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7256	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7257	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7258	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7259	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7260	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7261	#endif
7262	default:
7263	{
7264	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7265	AssertRCReturn(rc, rc);
7266
7267	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7268	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7269	{
7270	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7271	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7272	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7273	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7274	AssertRCReturn(rc, rc);
7275	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7276	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7277	0 /* GCPtrFaultAddress */);
7278	AssertRCReturn(rc, rc);
7279	}
7280	else
7281	{
7282	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7283	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7284	}
7285	break;
7286	}
7287	}
7288	break;
7289	}
7290
7291	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7292	default:
7293	{
7294	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7295	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7296	break;
7297	}
7298	}
7299	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7300	return rc;
7301	}
7302
7303
7304	/**
7305	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7306	*/
7307	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7308	{
7309	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7310
7311	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7312	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7313	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7314	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7315	AssertRCReturn(rc, rc);
7316
7317	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7318	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7319	return VINF_SUCCESS;
7320	}
7321
7322
7323	/**
7324	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7325	*/
7326	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7327	{
7328	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7329	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7330	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7331	}
7332
7333
7334	/**
7335	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7336	*/
7337	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7338	{
7339	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7340	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7341	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7342	}
7343
7344
7345	/**
7346	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7347	*/
7348	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7349	{
7350	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7351	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7352	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7353	}
7354
7355
7356	/**
7357	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7358	*/
7359	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7360	{
7361	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7362	PVM pVM = pVCpu->CTX_SUFF(pVM);
7363	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7364	if (RT_LIKELY(rc == VINF_SUCCESS))
7365	{
7366	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7367	Assert(pVmxTransient->cbInstr == 2);
7368	}
7369	else
7370	{
7371	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7372	rc = VERR_EM_INTERPRETER;
7373	}
7374	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7375	return rc;
7376	}
7377
7378
7379	/**
7380	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7381	*/
7382	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7383	{
7384	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7385	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7386	AssertRCReturn(rc, rc);
7387
7388	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7389	return VINF_EM_RAW_EMULATE_INSTR;
7390
7391	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7392	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7393	}
7394
7395
7396	/**
7397	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7398	*/
7399	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7400	{
7401	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7402	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7403	AssertRCReturn(rc, rc);
7404
7405	PVM pVM = pVCpu->CTX_SUFF(pVM);
7406	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7407	if (RT_LIKELY(rc == VINF_SUCCESS))
7408	{
7409	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7410	Assert(pVmxTransient->cbInstr == 2);
7411	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7412	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7413	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7414	}
7415	else
7416	{
7417	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7418	rc = VERR_EM_INTERPRETER;
7419	}
7420	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7421	return rc;
7422	}
7423
7424
7425	/**
7426	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7427	*/
7428	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7429	{
7430	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7431	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7432	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7433	AssertRCReturn(rc, rc);
7434
7435	PVM pVM = pVCpu->CTX_SUFF(pVM);
7436	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7437	if (RT_LIKELY(rc == VINF_SUCCESS))
7438	{
7439	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7440	Assert(pVmxTransient->cbInstr == 3);
7441	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7442	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7443	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7444	}
7445	else
7446	{
7447	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7448	rc = VERR_EM_INTERPRETER;
7449	}
7450	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7451	return rc;
7452	}
7453
7454
7455	/**
7456	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7457	*/
7458	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7459	{
7460	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7461	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7462	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7463	AssertRCReturn(rc, rc);
7464
7465	PVM pVM = pVCpu->CTX_SUFF(pVM);
7466	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7467	if (RT_LIKELY(rc == VINF_SUCCESS))
7468	{
7469	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7470	Assert(pVmxTransient->cbInstr == 2);
7471	}
7472	else
7473	{
7474	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7475	rc = VERR_EM_INTERPRETER;
7476	}
7477	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7478	return rc;
7479	}
7480
7481
7482	/**
7483	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7484	*/
7485	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7486	{
7487	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7488	PVM pVM = pVCpu->CTX_SUFF(pVM);
7489	Assert(!pVM->hm.s.fNestedPaging);
7490
7491	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7492	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7493	AssertRCReturn(rc, rc);
7494
7495	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7496	rc = VBOXSTRICTRC_VAL(rc2);
7497	if (RT_LIKELY(rc == VINF_SUCCESS))
7498	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7499	else
7500	{
7501	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
7502	pVmxTransient->uExitQualification, rc));
7503	}
7504	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7505	return rc;
7506	}
7507
7508
7509	/**
7510	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7511	*/
7512	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7513	{
7514	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7515	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7516	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7517	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7518	AssertRCReturn(rc, rc);
7519
7520	PVM pVM = pVCpu->CTX_SUFF(pVM);
7521	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7522	if (RT_LIKELY(rc == VINF_SUCCESS))
7523	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7524	else
7525	{
7526	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7527	rc = VERR_EM_INTERPRETER;
7528	}
7529	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7530	return rc;
7531	}
7532
7533
7534	/**
7535	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7536	*/
7537	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7538	{
7539	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7540	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7541	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7542	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7543	AssertRCReturn(rc, rc);
7544
7545	PVM pVM = pVCpu->CTX_SUFF(pVM);
7546	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7547	rc = VBOXSTRICTRC_VAL(rc2);
7548	if (RT_LIKELY( rc == VINF_SUCCESS
7549	\|\| rc == VINF_EM_HALT))
7550	{
7551	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7552	AssertRCReturn(rc3, rc3);
7553
7554	if ( rc == VINF_EM_HALT
7555	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7556	{
7557	rc = VINF_SUCCESS;
7558	}
7559	}
7560	else
7561	{
7562	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7563	rc = VERR_EM_INTERPRETER;
7564	}
7565	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7566	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7567	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7568	return rc;
7569	}
7570
7571
7572	/**
7573	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7574	*/
7575	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7576	{
7577	/*
7578	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7579	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7580	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7581	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7582	*/
7583	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7584	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7585	}
7586
7587
7588	/**
7589	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7590	*/
7591	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7592	{
7593	/*
7594	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7595	* root operation. If we get there there is something funny going on.
7596	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7597	*/
7598	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7599	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7600	}
7601
7602
7603	/**
7604	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7605	*/
7606	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7607	{
7608	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7609	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7610	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7611	}
7612
7613
7614	/**
7615	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7616	*/
7617	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7618	{
7619	/*
7620	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7621	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7622	* See Intel spec. 25.3 "Other Causes of VM-exits".
7623	*/
7624	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7625	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7626	}
7627
7628
7629	/**
7630	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7631	* VM-exit.
7632	*/
7633	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7634	{
7635	/*
7636	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7637	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7638	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7639	*/
7640	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7641	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7642	}
7643
7644
7645	/**
7646	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7647	* VM-exit.
7648	*/
7649	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7650	{
7651	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7652	return VINF_EM_RESET;
7653	}
7654
7655
7656	/**
7657	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7658	*/
7659	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7660	{
7661	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7662	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7663	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7664	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7665	AssertRCReturn(rc, rc);
7666
7667	pMixedCtx->rip++;
7668	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7669	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7670	rc = VINF_SUCCESS;
7671	else
7672	rc = VINF_EM_HALT;
7673
7674	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7675	return rc;
7676	}
7677
7678
7679	/**
7680	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7681	*/
7682	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7683	{
7684	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7685	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7686	return VINF_SUCCESS;
7687	}
7688
7689
7690	/**
7691	* VM-exit handler for expiry of the VMX preemption timer.
7692	*/
7693	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7694	{
7695	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7696
7697	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7698	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7699
7700	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7701	PVM pVM = pVCpu->CTX_SUFF(pVM);
7702	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7703	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7704	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7705	}
7706
7707
7708	/**
7709	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7710	*/
7711	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7712	{
7713	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7714
7715	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7716	/** @todo check if XSETBV is supported by the recompiler. */
7717	return VERR_EM_INTERPRETER;
7718	}
7719
7720
7721	/**
7722	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7723	*/
7724	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7725	{
7726	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7727
7728	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7729	/** @todo implement EMInterpretInvpcid() */
7730	return VERR_EM_INTERPRETER;
7731	}
7732
7733
7734	/**
7735	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7736	* Error VM-exit.
7737	*/
7738	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7739	{
7740	uint32_t uIntrState;
7741	HMVMXHCUINTREG uHCReg;
7742	uint64_t u64Val;
7743	uint32_t u32Val;
7744
7745	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7746	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7747	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7748	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7749	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7750	AssertRCReturn(rc, rc);
7751
7752	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7753	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7754	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7755	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7756
7757	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7758	Log(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7759	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7760	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7761	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7762	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7763	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7764	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7765	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7766	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7767	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7768	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7769
7770	PVM pVM = pVCpu->CTX_SUFF(pVM);
7771	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7772
7773	return VERR_VMX_INVALID_GUEST_STATE;
7774	}
7775
7776
7777	/**
7778	* VM-exit handler for VM-entry failure due to an MSR-load
7779	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7780	*/
7781	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7782	{
7783	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7784	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7785	}
7786
7787
7788	/**
7789	* VM-exit handler for VM-entry failure due to a machine-check event
7790	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7791	*/
7792	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7793	{
7794	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7795	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7796	}
7797
7798
7799	/**
7800	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7801	* theory.
7802	*/
7803	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7804	{
7805	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7806	return VERR_VMX_UNDEFINED_EXIT_CODE;
7807	}
7808
7809
7810	/**
7811	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7812	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7813	* Conditional VM-exit.
7814	*/
7815	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7816	{
7817	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7818
7819	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7820	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7821	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7822	return VERR_EM_INTERPRETER;
7823	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7824	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7825	}
7826
7827
7828	/**
7829	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7830	*/
7831	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7832	{
7833	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7834
7835	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7836	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7837	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7838	return VERR_EM_INTERPRETER;
7839	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7840	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7841	}
7842
7843
7844	/**
7845	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7846	*/
7847	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7848	{
7849	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7850
7851	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7852	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7853	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7854	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7855	AssertRCReturn(rc, rc);
7856
7857	PVM pVM = pVCpu->CTX_SUFF(pVM);
7858	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7859	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7860	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7861	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7862
7863	if (RT_LIKELY(rc == VINF_SUCCESS))
7864	{
7865	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7866	Assert(pVmxTransient->cbInstr == 2);
7867	}
7868	return rc;
7869	}
7870
7871
7872	/**
7873	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7874	*/
7875	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7876	{
7877	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7878	PVM pVM = pVCpu->CTX_SUFF(pVM);
7879	int rc = VINF_SUCCESS;
7880
7881	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7882	if ( pVM->hm.s.fTPRPatchingActive
7883	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7884	{
7885	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7886	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7887	{
7888	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7889	AssertRC(rc);
7890	}
7891
7892	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7893	Assert(pVmxTransient->cbInstr == 2);
7894	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7895	return VINF_SUCCESS;
7896	}
7897
7898	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7899	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7900	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7901	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7902	AssertRCReturn(rc, rc);
7903	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7904
7905	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7906	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7907	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7908
7909	if (RT_LIKELY(rc == VINF_SUCCESS))
7910	{
7911	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7912
7913	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7914	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7915	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7916	{
7917	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7918	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7919	}
7920	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7921	{
7922	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7923	AssertRCReturn(rc, rc);
7924	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7925	}
7926	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7927	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7928
7929	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7930	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
7931	{
7932	switch (pMixedCtx->ecx)
7933	{
7934	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7935	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7936	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7937	case MSR_K8_FS_BASE: /* no break */
7938	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7939	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
7940	}
7941	}
7942	#ifdef VBOX_STRICT
7943	else
7944	{
7945	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7946	switch (pMixedCtx->ecx)
7947	{
7948	case MSR_IA32_SYSENTER_CS:
7949	case MSR_IA32_SYSENTER_EIP:
7950	case MSR_IA32_SYSENTER_ESP:
7951	case MSR_K8_FS_BASE:
7952	case MSR_K8_GS_BASE:
7953	{
7954	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7955	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7956	}
7957
7958	case MSR_K8_LSTAR:
7959	case MSR_K6_STAR:
7960	case MSR_K8_SF_MASK:
7961	case MSR_K8_TSC_AUX:
7962	case MSR_K8_KERNEL_GS_BASE:
7963	{
7964	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7965	pMixedCtx->ecx));
7966	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7967	}
7968	}
7969	}
7970	#endif /* VBOX_STRICT */
7971	}
7972	return rc;
7973	}
7974
7975
7976	/**
7977	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7978	*/
7979	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7980	{
7981	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7982
7983	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
7984	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7985	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
7986	return VERR_EM_INTERPRETER;
7987	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7988	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7989	}
7990
7991
7992	/**
7993	* VM-exit handler for when the TPR value is lowered below the specified
7994	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7995	*/
7996	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7997	{
7998	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7999	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
8000
8001	/*
8002	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
8003	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
8004	* resume guest execution.
8005	*/
8006	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8007	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
8008	return VINF_SUCCESS;
8009	}
8010
8011
8012	/**
8013	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
8014	* VM-exit.
8015	*
8016	* @retval VINF_SUCCESS when guest execution can continue.
8017	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
8018	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
8019	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
8020	* recompiler.
8021	*/
8022	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8023	{
8024	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8025	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
8026	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8027	AssertRCReturn(rc, rc);
8028
8029	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
8030	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
8031	PVM pVM = pVCpu->CTX_SUFF(pVM);
8032	switch (uAccessType)
8033	{
8034	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
8035	{
8036	#if 0
8037	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
8038	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8039	#else
8040	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8041	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8042	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8043	#endif
8044	AssertRCReturn(rc, rc);
8045
8046	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8047	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
8048	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
8049	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
8050
8051	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
8052	{
8053	case 0: /* CR0 */
8054	Log(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
8055	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8056	break;
8057	case 2: /* C2 **/
8058	/* Nothing to do here, CR2 it's not part of the VMCS. */
8059	break;
8060	case 3: /* CR3 */
8061	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
8062	Log(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
8063	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
8064	break;
8065	case 4: /* CR4 */
8066	Log(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
8067	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
8068	break;
8069	case 8: /* CR8 */
8070	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8071	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
8072	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8073	break;
8074	default:
8075	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
8076	break;
8077	}
8078
8079	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8080	break;
8081	}
8082
8083	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
8084	{
8085	/* EMInterpretCRxRead() requires EFER MSR, CS. */
8086	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8087	AssertRCReturn(rc, rc);
8088	Assert( !pVM->hm.s.fNestedPaging
8089	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
8090	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
8091
8092	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8093	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
8094	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8095
8096	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8097	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
8098	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
8099	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8100	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8101	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
8102	break;
8103	}
8104
8105	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
8106	{
8107	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8108	AssertRCReturn(rc, rc);
8109	rc = EMInterpretCLTS(pVM, pVCpu);
8110	AssertRCReturn(rc, rc);
8111	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8112	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
8113	Log(("CRX CLTS write rc=%d\n", rc));
8114	break;
8115	}
8116
8117	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
8118	{
8119	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8120	AssertRCReturn(rc, rc);
8121	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
8122	if (RT_LIKELY(rc == VINF_SUCCESS))
8123	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8124	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
8125	Log(("CRX LMSW write rc=%d\n", rc));
8126	break;
8127	}
8128
8129	default:
8130	{
8131	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
8132	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8133	}
8134	}
8135
8136	/* Validate possible error codes. */
8137	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
8138	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8139	if (RT_SUCCESS(rc))
8140	{
8141	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8142	AssertRCReturn(rc2, rc2);
8143	}
8144
8145	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8146	return rc;
8147	}
8148
8149
8150	/**
8151	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8152	* VM-exit.
8153	*/
8154	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8155	{
8156	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8157	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8158
8159	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8160	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8161	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8162	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8163	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8164	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8165	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8166	AssertRCReturn(rc, rc);
8167	Log(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8168
8169	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8170	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8171	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8172	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8173	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8174	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
8175	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
8176
8177	/* I/O operation lookup arrays. */
8178	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
8179	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8180
8181	const uint32_t cbSize = s_aIOSize[uIOWidth];
8182	const uint32_t cbInstr = pVmxTransient->cbInstr;
8183	PVM pVM = pVCpu->CTX_SUFF(pVM);
8184	if (fIOString)
8185	{
8186	/* INS/OUTS - I/O String instruction. */
8187	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8188	/** @todo for now manually disassemble later optimize by getting the fields from
8189	* the VMCS. */
8190	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8191	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8192	* segment prefix info. */
8193	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8194	if (RT_SUCCESS(rc))
8195	{
8196	if (fIOWrite)
8197	{
8198	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8199	(DISCPUMODE)pDis->uAddrMode, cbSize);
8200	rc = VBOXSTRICTRC_VAL(rc2);
8201	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8202	}
8203	else
8204	{
8205	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8206	(DISCPUMODE)pDis->uAddrMode, cbSize);
8207	rc = VBOXSTRICTRC_VAL(rc2);
8208	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8209	}
8210	}
8211	else
8212	{
8213	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8214	rc = VINF_EM_RAW_EMULATE_INSTR;
8215	}
8216	}
8217	else
8218	{
8219	/* IN/OUT - I/O instruction. */
8220	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8221	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8222	if (fIOWrite)
8223	{
8224	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8225	rc = VBOXSTRICTRC_VAL(rc2);
8226	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8227	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8228	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8229	}
8230	else
8231	{
8232	uint32_t u32Result = 0;
8233	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8234	rc = VBOXSTRICTRC_VAL(rc2);
8235	if (IOM_SUCCESS(rc))
8236	{
8237	/* Save result of I/O IN instr. in AL/AX/EAX. */
8238	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8239	}
8240	else if (rc == VINF_IOM_R3_IOPORT_READ)
8241	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8242	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8243	}
8244	}
8245
8246	if (IOM_SUCCESS(rc))
8247	{
8248	pMixedCtx->rip += cbInstr;
8249	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8250	if (RT_LIKELY(rc == VINF_SUCCESS))
8251	{
8252	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8253	AssertRCReturn(rc, rc);
8254
8255	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8256	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8257	{
8258	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8259	for (unsigned i = 0; i < 4; i++)
8260	{
8261	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8262	if ( ( uIOPort >= pMixedCtx->dr[i]
8263	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8264	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8265	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8266	{
8267	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8268	uint64_t uDR6 = ASMGetDR6();
8269
8270	/* Clear all breakpoint status flags and set the one we just hit. */
8271	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8272	uDR6 \|= (uint64_t)RT_BIT(i);
8273
8274	/*
8275	* Note: AMD64 Architecture Programmer's Manual 13.1:
8276	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8277	* be cleared by software after the contents have been read.
8278	*/
8279	ASMSetDR6(uDR6);
8280
8281	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8282	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8283
8284	/* Paranoia. */
8285	pMixedCtx->dr[7] &= UINT64_C(0xffffffff); /* Upper 32 bits MBZ. */
8286	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8287	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8288
8289	/* Resync DR7 */
8290	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8291	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8292
8293	/* Set #DB to be injected into the VM and continue guest execution. */
8294	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8295	break;
8296	}
8297	}
8298	}
8299	}
8300	}
8301
8302	#ifdef DEBUG
8303	if (rc == VINF_IOM_R3_IOPORT_READ)
8304	Assert(!fIOWrite);
8305	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8306	Assert(fIOWrite);
8307	else
8308	{
8309	AssertMsg( RT_FAILURE(rc)
8310	\|\| rc == VINF_SUCCESS
8311	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8312	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8313	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8314	}
8315	#endif
8316
8317	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8318	return rc;
8319	}
8320
8321
8322	/**
8323	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8324	* VM-exit.
8325	*/
8326	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8327	{
8328	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8329
8330	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8331	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8332	AssertRCReturn(rc, rc);
8333	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8334	{
8335	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8336	AssertRCReturn(rc, rc);
8337	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8338	{
8339	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8340	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8341	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8342	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8343	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8344	{
8345	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8346	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8347
8348	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8349	Assert(!pVCpu->hm.s.Event.fPending);
8350	pVCpu->hm.s.Event.fPending = true;
8351	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8352	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8353	AssertRCReturn(rc, rc);
8354	if (fErrorCodeValid)
8355	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8356	else
8357	pVCpu->hm.s.Event.u32ErrCode = 0;
8358	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8359	&& uVector == X86_XCPT_PF)
8360	{
8361	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8362	}
8363	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8364	}
8365	}
8366	}
8367	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8368	* emulation. */
8369	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8370	return VERR_EM_INTERPRETER;
8371	}
8372
8373
8374	/**
8375	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8376	*/
8377	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8378	{
8379	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8380	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8381	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8382	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8383	AssertRCReturn(rc, rc);
8384	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8385	return VINF_EM_DBG_STOP;
8386	}
8387
8388
8389	/**
8390	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8391	*/
8392	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8393	{
8394	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8395
8396	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8397	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8398	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8399	return VINF_SUCCESS;
8400	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8401	return rc;
8402
8403	#if 0
8404	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8405	* just sync the whole thing. */
8406	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8407	#else
8408	/* Aggressive state sync. for now. */
8409	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8410	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8411	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8412	#endif
8413	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8414	AssertRCReturn(rc, rc);
8415
8416	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8417	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8418	switch (uAccessType)
8419	{
8420	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8421	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8422	{
8423	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8424	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8425	{
8426	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8427	}
8428
8429	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8430	GCPhys &= PAGE_BASE_GC_MASK;
8431	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8432	PVM pVM = pVCpu->CTX_SUFF(pVM);
8433	Log(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
8434	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8435
8436	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8437	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8438	CPUMCTX2CORE(pMixedCtx), GCPhys);
8439	rc = VBOXSTRICTRC_VAL(rc2);
8440	Log(("ApicAccess rc=%d\n", rc));
8441	if ( rc == VINF_SUCCESS
8442	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8443	\|\| rc == VERR_PAGE_NOT_PRESENT)
8444	{
8445	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8446	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8447	rc = VINF_SUCCESS;
8448	}
8449	break;
8450	}
8451
8452	default:
8453	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8454	rc = VINF_EM_RAW_EMULATE_INSTR;
8455	break;
8456	}
8457
8458	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8459	return rc;
8460	}
8461
8462
8463	/**
8464	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8465	* VM-exit.
8466	*/
8467	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8468	{
8469	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8470
8471	/* We should -not- get this VM-exit if the guest is debugging. */
8472	if (CPUMIsGuestDebugStateActive(pVCpu))
8473	{
8474	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8475	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8476	}
8477
8478	int rc = VERR_INTERNAL_ERROR_5;
8479	if ( !DBGFIsStepping(pVCpu)
8480	&& !CPUMIsHyperDebugStateActive(pVCpu))
8481	{
8482	/* Don't intercept MOV DRx. */
8483	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8484	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8485	AssertRCReturn(rc, rc);
8486
8487	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8488	PVM pVM = pVCpu->CTX_SUFF(pVM);
8489	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8490	AssertRC(rc);
8491	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8492
8493	#ifdef VBOX_WITH_STATISTICS
8494	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8495	AssertRCReturn(rc, rc);
8496	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8497	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8498	else
8499	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8500	#endif
8501	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8502	return VINF_SUCCESS;
8503	}
8504
8505	/*
8506	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8507	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8508	*/
8509	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8510	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8511	AssertRCReturn(rc, rc);
8512
8513	PVM pVM = pVCpu->CTX_SUFF(pVM);
8514	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8515	{
8516	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8517	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8518	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8519	if (RT_SUCCESS(rc))
8520	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8521	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8522	}
8523	else
8524	{
8525	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8526	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8527	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8528	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8529	}
8530
8531	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8532	if (RT_SUCCESS(rc))
8533	{
8534	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8535	AssertRCReturn(rc2, rc2);
8536	}
8537	return rc;
8538	}
8539
8540
8541	/**
8542	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8543	* Conditional VM-exit.
8544	*/
8545	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8546	{
8547	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8548	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8549
8550	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8551	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8552	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8553	return VINF_SUCCESS;
8554	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8555	return rc;
8556
8557	RTGCPHYS GCPhys = 0;
8558	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8559
8560	#if 0
8561	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8562	#else
8563	/* Aggressive state sync. for now. */
8564	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8565	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8566	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8567	#endif
8568	AssertRCReturn(rc, rc);
8569
8570	/*
8571	* If we succeed, resume guest execution.
8572	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8573	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8574	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8575	* weird case. See @bugref{6043}.
8576	*/
8577	PVM pVM = pVCpu->CTX_SUFF(pVM);
8578	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8579	rc = VBOXSTRICTRC_VAL(rc2);
8580	Log(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8581	if ( rc == VINF_SUCCESS
8582	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8583	\|\| rc == VERR_PAGE_NOT_PRESENT)
8584	{
8585	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8586	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8587	return VINF_SUCCESS;
8588	}
8589	return rc;
8590	}
8591
8592
8593	/**
8594	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8595	* VM-exit.
8596	*/
8597	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8598	{
8599	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8600	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8601
8602	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8603	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8604	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8605	return VINF_SUCCESS;
8606	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8607	return rc;
8608
8609	RTGCPHYS GCPhys = 0;
8610	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8611	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8612	#if 0
8613	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8614	#else
8615	/* Aggressive state sync. for now. */
8616	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8617	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8618	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8619	#endif
8620	AssertRCReturn(rc, rc);
8621
8622	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8623	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
8624
8625	RTGCUINT uErrorCode = 0;
8626	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8627	uErrorCode \|= X86_TRAP_PF_ID;
8628	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8629	uErrorCode \|= X86_TRAP_PF_RW;
8630	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8631	uErrorCode \|= X86_TRAP_PF_P;
8632
8633	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8634
8635	Log(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
8636	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8637
8638	/* Handle the pagefault trap for the nested shadow table. */
8639	PVM pVM = pVCpu->CTX_SUFF(pVM);
8640	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8641	TRPMResetTrap(pVCpu);
8642
8643	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8644	if ( rc == VINF_SUCCESS
8645	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8646	\|\| rc == VERR_PAGE_NOT_PRESENT)
8647	{
8648	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8649	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8650	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8651	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8652	return VINF_SUCCESS;
8653	}
8654
8655	Log(("EPT return to ring-3 rc=%d\n"));
8656	return rc;
8657	}
8658
8659
8660	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8661	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8662	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8663	/**
8664	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8665	*/
8666	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8667	{
8668	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8669	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8670
8671	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8672	AssertRCReturn(rc, rc);
8673
8674	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8675	{
8676	/* Old-style FPU error reporting needs some extra work. */
8677	/** @todo don't fall back to the recompiler, but do it manually. */
8678	return VERR_EM_INTERPRETER;
8679	}
8680	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8681	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8682	return rc;
8683	}
8684
8685
8686	/**
8687	* VM-exit exception handler for #BP (Breakpoint exception).
8688	*/
8689	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8690	{
8691	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8692	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8693
8694	/** @todo Try optimize this by not saving the entire guest state unless
8695	* really needed. */
8696	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8697	AssertRCReturn(rc, rc);
8698
8699	PVM pVM = pVCpu->CTX_SUFF(pVM);
8700	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8701	if (rc == VINF_EM_RAW_GUEST_TRAP)
8702	{
8703	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8704	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8705	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8706	AssertRCReturn(rc, rc);
8707
8708	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8709	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8710	}
8711
8712	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8713	return rc;
8714	}
8715
8716
8717	/**
8718	* VM-exit exception handler for #DB (Debug exception).
8719	*/
8720	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8721	{
8722	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8723	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8724
8725	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8726	AssertRCReturn(rc, rc);
8727
8728	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8729	uint64_t uDR6 = X86_DR6_INIT_VAL;
8730	uDR6 \|= (pVmxTransient->uExitQualification
8731	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8732	PVM pVM = pVCpu->CTX_SUFF(pVM);
8733	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8734	if (rc == VINF_EM_RAW_GUEST_TRAP)
8735	{
8736	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8737	pMixedCtx->dr[6] = uDR6;
8738
8739	if (CPUMIsGuestDebugStateActive(pVCpu))
8740	ASMSetDR6(pMixedCtx->dr[6]);
8741
8742	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8743
8744	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8745	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8746
8747	/* Paranoia. */
8748	pMixedCtx->dr[7] &= UINT64_C(0xffffffff); /* upper 32 bits MBZ. */
8749	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8750	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8751
8752	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
8753	AssertRCReturn(rc,rc);
8754
8755	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8756	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8757	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8758	AssertRCReturn(rc2, rc2);
8759	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8760	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8761	rc = VINF_SUCCESS;
8762	}
8763
8764	return rc;
8765	}
8766
8767
8768	/**
8769	* VM-exit exception handler for #NM (Device-not-available exception: floating
8770	* point exception).
8771	*/
8772	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8773	{
8774	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8775
8776	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8777	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8778	#endif
8779
8780	/* We require CR0 and EFER. EFER is always up-to-date. */
8781	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8782	AssertRCReturn(rc, rc);
8783
8784	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8785	PVM pVM = pVCpu->CTX_SUFF(pVM);
8786	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8787	if (rc == VINF_SUCCESS)
8788	{
8789	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8790	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8791	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8792	return VINF_SUCCESS;
8793	}
8794
8795	/* Forward #NM to the guest. */
8796	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8797	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8798	AssertRCReturn(rc, rc);
8799	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8800	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8801	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8802	return rc;
8803	}
8804
8805
8806	/**
8807	* VM-exit exception handler for #GP (General-protection exception).
8808	*
8809	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8810	*/
8811	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8812	{
8813	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8814	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8815
8816	int rc = VERR_INTERNAL_ERROR_5;
8817	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8818	{
8819	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8820	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8821	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8822	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8823	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8824	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8825	AssertRCReturn(rc, rc);
8826	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8827	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8828	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8829	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8830	return rc;
8831	#else
8832	/* We don't intercept #GP. */
8833	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8834	return VERR_VMX_UNEXPECTED_EXCEPTION;
8835	#endif
8836	}
8837
8838	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8839	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8840
8841	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8842	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8843	AssertRCReturn(rc, rc);
8844
8845	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8846	uint32_t cbOp = 0;
8847	PVM pVM = pVCpu->CTX_SUFF(pVM);
8848	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8849	if (RT_SUCCESS(rc))
8850	{
8851	rc = VINF_SUCCESS;
8852	Assert(cbOp == pDis->cbInstr);
8853	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8854	switch (pDis->pCurInstr->uOpcode)
8855	{
8856	case OP_CLI:
8857	pMixedCtx->eflags.Bits.u1IF = 0;
8858	pMixedCtx->rip += pDis->cbInstr;
8859	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8860	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8861	break;
8862
8863	case OP_STI:
8864	pMixedCtx->eflags.Bits.u1IF = 1;
8865	pMixedCtx->rip += pDis->cbInstr;
8866	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8867	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8868	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8869	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8870	break;
8871
8872	case OP_HLT:
8873	rc = VINF_EM_HALT;
8874	pMixedCtx->rip += pDis->cbInstr;
8875	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8876	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8877	break;
8878
8879	case OP_POPF:
8880	{
8881	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8882	uint32_t cbParm = 0;
8883	uint32_t uMask = 0;
8884	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8885	{
8886	cbParm = 4;
8887	uMask = 0xffffffff;
8888	}
8889	else
8890	{
8891	cbParm = 2;
8892	uMask = 0xffff;
8893	}
8894
8895	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8896	RTGCPTR GCPtrStack = 0;
8897	X86EFLAGS uEflags;
8898	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8899	&GCPtrStack);
8900	if (RT_SUCCESS(rc))
8901	{
8902	Assert(sizeof(uEflags.u32) >= cbParm);
8903	uEflags.u32 = 0;
8904	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8905	}
8906	if (RT_FAILURE(rc))
8907	{
8908	rc = VERR_EM_INTERPRETER;
8909	break;
8910	}
8911	Log(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8912	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8913	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8914	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8915	pMixedCtx->eflags.Bits.u1RF = 0;
8916	pMixedCtx->esp += cbParm;
8917	pMixedCtx->esp &= uMask;
8918	pMixedCtx->rip += pDis->cbInstr;
8919	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8920	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8921	break;
8922	}
8923
8924	case OP_PUSHF:
8925	{
8926	uint32_t cbParm = 0;
8927	uint32_t uMask = 0;
8928	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8929	{
8930	cbParm = 4;
8931	uMask = 0xffffffff;
8932	}
8933	else
8934	{
8935	cbParm = 2;
8936	uMask = 0xffff;
8937	}
8938
8939	/* Get the stack pointer & push the contents of eflags onto the stack. */
8940	RTGCPTR GCPtrStack = 0;
8941	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8942	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8943	if (RT_FAILURE(rc))
8944	{
8945	rc = VERR_EM_INTERPRETER;
8946	break;
8947	}
8948	X86EFLAGS uEflags;
8949	uEflags = pMixedCtx->eflags;
8950	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8951	uEflags.Bits.u1RF = 0;
8952	uEflags.Bits.u1VM = 0;
8953
8954	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8955	if (RT_FAILURE(rc))
8956	{
8957	rc = VERR_EM_INTERPRETER;
8958	break;
8959	}
8960	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8961	pMixedCtx->esp -= cbParm;
8962	pMixedCtx->esp &= uMask;
8963	pMixedCtx->rip += pDis->cbInstr;
8964	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8965	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8966	break;
8967	}
8968
8969	case OP_IRET:
8970	{
8971	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8972	* instruction reference. */
8973	RTGCPTR GCPtrStack = 0;
8974	uint32_t uMask = 0xffff;
8975	uint16_t aIretFrame[3];
8976	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8977	{
8978	rc = VERR_EM_INTERPRETER;
8979	break;
8980	}
8981	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8982	&GCPtrStack);
8983	if (RT_SUCCESS(rc))
8984	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8985	if (RT_FAILURE(rc))
8986	{
8987	rc = VERR_EM_INTERPRETER;
8988	break;
8989	}
8990	pMixedCtx->eip = 0;
8991	pMixedCtx->ip = aIretFrame[0];
8992	pMixedCtx->cs.Sel = aIretFrame[1];
8993	pMixedCtx->cs.ValidSel = aIretFrame[1];
8994	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8995	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8996	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8997	pMixedCtx->sp += sizeof(aIretFrame);
8998	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8999	\| HM_CHANGED_GUEST_RFLAGS;
9000	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
9001	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
9002	break;
9003	}
9004
9005	case OP_INT:
9006	{
9007	uint16_t uVector = pDis->Param1.uValue & 0xff;
9008	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
9009	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9010	break;
9011	}
9012
9013	case OP_INTO:
9014	{
9015	if (pMixedCtx->eflags.Bits.u1OF)
9016	{
9017	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
9018	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9019	}
9020	break;
9021	}
9022
9023	default:
9024	{
9025	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
9026	EMCODETYPE_SUPERVISOR);
9027	rc = VBOXSTRICTRC_VAL(rc2);
9028	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
9029	Log(("#GP rc=%Rrc\n", rc));
9030	break;
9031	}
9032	}
9033	}
9034	else
9035	rc = VERR_EM_INTERPRETER;
9036
9037	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
9038	("#GP Unexpected rc=%Rrc\n", rc));
9039	return rc;
9040	}
9041
9042
9043	/**
9044	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
9045	* the exception reported in the VMX transient structure back into the VM.
9046	*
9047	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
9048	* up-to-date.
9049	*/
9050	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9051	{
9052	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9053
9054	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
9055	hmR0VmxCheckExitDueToEventDelivery(). */
9056	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9057	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9058	AssertRCReturn(rc, rc);
9059	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
9060	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9061	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9062	return VINF_SUCCESS;
9063	}
9064
9065
9066	/**
9067	* VM-exit exception handler for #PF (Page-fault exception).
9068	*/
9069	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9070	{
9071	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9072	PVM pVM = pVCpu->CTX_SUFF(pVM);
9073	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9074	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9075	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9076	AssertRCReturn(rc, rc);
9077
9078	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
9079	if (pVM->hm.s.fNestedPaging)
9080	{
9081	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
9082	{
9083	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9084	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9085	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9086	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
9087	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9088	}
9089	else
9090	{
9091	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9092	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
9093	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9094	Log(("Pending #DF due to vectoring #PF. NP\n"));
9095	}
9096	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9097	return rc;
9098	}
9099	#else
9100	Assert(!pVM->hm.s.fNestedPaging);
9101	#endif
9102
9103	#ifdef VBOX_HM_WITH_GUEST_PATCHING
9104	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9105	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9106	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9107	AssertRCReturn(rc, rc);
9108	/* Shortcut for APIC TPR access, only for 32-bit guests. */
9109	if ( pVM->hm.s.fTRPPatchingAllowed
9110	&& pVM->hm.s.pGuestPatchMem
9111	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
9112	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
9113	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
9114	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
9115	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
9116	{
9117	RTGCPHYS GCPhys;
9118	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
9119	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
9120	if ( rc == VINF_SUCCESS
9121	&& GCPhys == GCPhysApicBase)
9122	{
9123	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9124	AssertRCReturn(rc, rc);
9125
9126	/* Only attempt to patch the instruction once. */
9127	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
9128	if (!pPatch)
9129	return VINF_EM_HM_PATCH_TPR_INSTR;
9130	}
9131	}
9132	#endif
9133
9134	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9135	AssertRCReturn(rc, rc);
9136
9137	Log(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
9138	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9139
9140	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9141	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9142	(RTGCPTR)pVmxTransient->uExitQualification);
9143
9144	Log(("#PF: rc=%Rrc\n", rc));
9145	if (rc == VINF_SUCCESS)
9146	{
9147	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9148	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9149	* memory? We don't update the whole state here... */
9150	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9151	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9152	TRPMResetTrap(pVCpu);
9153	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9154	return rc;
9155	}
9156	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9157	{
9158	if (!pVmxTransient->fVectoringPF)
9159	{
9160	/* It's a guest page fault and needs to be reflected to the guest. */
9161	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9162	TRPMResetTrap(pVCpu);
9163	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9164	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9165	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9166	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9167	}
9168	else
9169	{
9170	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9171	TRPMResetTrap(pVCpu);
9172	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
9173	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9174	Log(("#PF: Pending #DF due to vectoring #PF\n"));
9175	}
9176
9177	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9178	return VINF_SUCCESS;
9179	}
9180
9181	TRPMResetTrap(pVCpu);
9182	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9183	return rc;
9184	}
9185

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 46286

Download in other formats: