HMVMXR0.cpp@ 45467

Last change on this file since 45467 was 45467, checked in by vboxsync, 12 years ago
VMM/VMMR0/HM: bits.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 368.9 KB

Line
1	/* $Id: HMVMXR0.cpp 45467 2013-04-10 16:57:41Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
39	#endif
40
41	/*******************************************************************************
42	* Defined Constants And Macros *
43	*******************************************************************************/
44	#if defined(RT_ARCH_AMD64)
45	# define VMX_IS_64BIT_HOST_MODE() (true)
46	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
47	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
48	#else
49	# define VMX_IS_64BIT_HOST_MODE() (false)
50	#endif
51
52	#define VMX_SEL_UNUSABLE RT_BIT(16)
53
54	/**
55	* Updated-guest-state flags.
56	*/
57	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
58	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
59	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
60	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
61	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
62	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
63	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
64	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
65	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
66	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
67	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
68	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
69	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
70	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
71	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
72	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
73	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
74	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
75	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(18)
76	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(19)
77	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(20)
78	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
79	\| VMX_UPDATED_GUEST_RIP \
80	\| VMX_UPDATED_GUEST_RSP \
81	\| VMX_UPDATED_GUEST_RFLAGS \
82	\| VMX_UPDATED_GUEST_CR0 \
83	\| VMX_UPDATED_GUEST_CR3 \
84	\| VMX_UPDATED_GUEST_CR4 \
85	\| VMX_UPDATED_GUEST_GDTR \
86	\| VMX_UPDATED_GUEST_IDTR \
87	\| VMX_UPDATED_GUEST_LDTR \
88	\| VMX_UPDATED_GUEST_TR \
89	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
90	\| VMX_UPDATED_GUEST_DEBUG \
91	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
92	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
93	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
94	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
95	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
96	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
97	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
98	\| VMX_UPDATED_GUEST_APIC_STATE)
99
100	/**
101	* Flags to skip redundant reads of some common VMCS fields.
102	*/
103	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
104	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
105	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
106	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
107	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
108	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
109
110	/*
111	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
112	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
113	* we have Nested Paging support.
114	*/
115	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
116	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
117	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
118	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
119	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
120	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
121	\| RT_BIT(X86_XCPT_XF))
122
123	/* Maximum VM-instruction error number. */
124	#define VMX_INSTR_ERROR_MAX 28
125
126	/*******************************************************************************
127	* Structures and Typedefs *
128	*******************************************************************************/
129	/**
130	* A state structure for holding miscellaneous information across
131	* VMX non-root operation and restored after the transition.
132	*/
133	typedef struct VMXTRANSIENT
134	{
135	/** The host's rflags/eflags. */
136	RTCCUINTREG uEFlags;
137	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
138	uint64_t u64LStarMsr;
139	/** The guest's TPR value used for TPR shadowing. */
140	uint8_t u8GuestTpr;
141
142	/** The basic VM-exit reason. */
143	uint16_t uExitReason;
144	/** The VM-exit exit qualification. */
145	RTGCUINTPTR uExitQualification;
146	/** The VM-exit interruption error code. */
147	uint32_t uExitIntrErrorCode;
148
149	/** The VM-exit interruption-information field. */
150	uint32_t uExitIntrInfo;
151	/** Whether the VM-entry failed or not. */
152	bool fVMEntryFailed;
153	/** The VM-exit instruction-length field. */
154	uint32_t cbInstr;
155
156	/** The VM-entry interruption-information field. */
157	uint32_t uEntryIntrInfo;
158	/** The VM-entry exception error code field. */
159	uint32_t uEntryXcptErrorCode;
160	/** The VM-entry instruction length field. */
161	uint32_t cbEntryInstr;
162
163	/** IDT-vectoring information field. */
164	uint32_t uIdtVectoringInfo;
165	/** IDT-vectoring error code. */
166	uint32_t uIdtVectoringErrorCode;
167
168	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
169	uint32_t fVmcsFieldsRead;
170	/** Whether TSC-offsetting should be setup before VM-entry. */
171	bool fUpdateTscOffsettingAndPreemptTimer;
172	/** Whether the VM-exit was caused by a page-fault during delivery of a
173	* contributary exception or a page-fault. */
174	bool fVectoringPF;
175	} VMXTRANSIENT, *PVMXTRANSIENT;
176
177	/**
178	* MSR-bitmap read permissions.
179	*/
180	typedef enum VMXMSREXITREAD
181	{
182	/** Reading this MSR causes a VM-exit. */
183	VMXMSREXIT_INTERCEPT_READ = 0xb,
184	/** Reading this MSR does not cause a VM-exit. */
185	VMXMSREXIT_PASSTHRU_READ
186	} VMXMSREXITREAD;
187
188	/**
189	* MSR-bitmap write permissions.
190	*/
191	typedef enum VMXMSREXITWRITE
192	{
193	/** Writing to this MSR causes a VM-exit. */
194	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
195	/** Writing to this MSR does not cause a VM-exit. */
196	VMXMSREXIT_PASSTHRU_WRITE
197	} VMXMSREXITWRITE;
198
199	/*******************************************************************************
200	* Internal Functions *
201	*******************************************************************************/
202	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
203	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
204	uint32_t cbInstr, uint32_t u32ErrCode);
205	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
206	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
207	#endif
208	#if 0
209	DECLINLINE(int) hmR0VmxHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
210	unsigned rcReason);
211	#endif
212
213	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
214	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
215	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
216	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
217	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
218	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
219	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
220	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
221	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
222	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
223	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
224	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264
265	/*******************************************************************************
266	* Global Variables *
267	*******************************************************************************/
268	/** @todo Move this to hm_vmx.h. */
269	/**
270	* VM-exit handler.
271	*
272	* @returns VBox status code.
273	* @param pVM Pointer to the VM.
274	* @param pVCpu Pointer to the VMCPU.
275	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
276	* out-of-sync. Make sure to update the required
277	* fields before using them.
278	* @param pVmxTransient Pointer to the VMX-transient structure.
279	*/
280	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	/** Pointer to VM-exit handler. */
282	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
283
284	static const PFNVMEXITHANDLER s_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
285	{
286	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
287	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
288	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
289	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
290	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
291	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
292	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
293	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
294	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
295	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
296	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
297	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
298	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
299	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
300	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
301	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
302	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
303	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
304	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitInjectXcptUD,
305	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitInjectXcptUD,
306	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitInjectXcptUD,
307	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitInjectXcptUD,
308	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitInjectXcptUD,
309	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitInjectXcptUD,
310	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitInjectXcptUD,
311	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitInjectXcptUD,
312	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitInjectXcptUD,
313	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitInjectXcptUD,
314	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
315	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
316	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
317	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
318	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
319	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
320	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
321	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
322	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
323	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
324	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
325	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
326	/* 40 UNDEFINED */ hmR0VmxExitPause,
327	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
328	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
329	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
330	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
331	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
332	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
333	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
334	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
335	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
336	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitInjectXcptUD,
337	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
338	/* 52 VMX_EXIT_PREEMPTION_TIMER */ hmR0VmxExitPreemptionTimer,
339	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInjectXcptUD,
340	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
341	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
342	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
343	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
344	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
345	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitInjectXcptUD
346	};
347
348	static const char* const s_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
349	{
350	/* 0 */ "(Not Used)",
351	/* 1 */ "VMCALL executed in VMX root operation.",
352	/* 2 */ "VMCLEAR with invalid physical address.",
353	/* 3 */ "VMCLEAR with VMXON pointer.",
354	/* 4 */ "VMLAUNCH with non-clear VMCS.",
355	/* 5 */ "VMRESUME with non-launched VMCS.",
356	/* 6 */ "VMRESUME after VMXOFF",
357	/* 7 */ "VM entry with invalid control fields.",
358	/* 8 */ "VM entry with invalid host state fields.",
359	/* 9 */ "VMPTRLD with invalid physical address.",
360	/* 10 */ "VMPTRLD with VMXON pointer.",
361	/* 11 */ "VMPTRLD with incorrect revision identifier.",
362	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
363	/* 13 */ "VMWRITE to read-only VMCS component.",
364	/* 14 */ "(Not Used)",
365	/* 15 */ "VMXON executed in VMX root operation.",
366	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
367	/* 17 */ "VM entry with non-launched executing VMCS.",
368	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
369	/* 19 */ "VMCALL with non-clear VMCS.",
370	/* 20 */ "VMCALL with invalid VM-exit control fields.",
371	/* 21 */ "(Not Used)",
372	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
373	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
374	/* 24 */ "VMCALL with invalid SMM-monitor features.",
375	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
376	/* 26 */ "VM entry with events blocked by MOV SS.",
377	/* 27 */ "(Not Used)",
378	/* 28 */ "Invalid operand to INVEPT/INVVPID."
379	};
380
381
382	/**
383	* Updates the VM's last error record. If there was a VMX instruction error,
384	* reads the error data from the VMCS and updates VCPU's last error record as
385	* well.
386	*
387	* @param pVM Pointer to the VM.
388	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
389	* VERR_VMX_GENERIC).
390	* @param rc The error code.
391	*/
392	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
393	{
394	AssertPtr(pVM);
395	if (rc == VERR_VMX_GENERIC)
396	{
397	AssertPtrReturnVoid(pVCpu);
398	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
399	}
400	pVM->hm.s.lLastError = rc;
401	}
402
403
404	/**
405	* Reads the VM-entry interruption-information field from the VMCS into the VMX
406	* transient structure.
407	*
408	* @returns VBox status code.
409	* @param pVmxTransient Pointer to the VMX transient structure.
410	*
411	* @remarks No-long-jump zone!!!
412	*/
413	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
414	{
415	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
416	AssertRCReturn(rc, rc);
417	return VINF_SUCCESS;
418	}
419
420
421	/**
422	* Reads the VM-entry exception error code field from the VMCS into
423	* the VMX transient structure.
424	*
425	* @returns VBox status code.
426	* @param pVmxTransient Pointer to the VMX transient structure.
427	*
428	* @remarks No-long-jump zone!!!
429	*/
430	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
431	{
432	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
433	AssertRCReturn(rc, rc);
434	return VINF_SUCCESS;
435	}
436
437
438	/**
439	* Reads the VM-entry exception error code field from the VMCS into
440	* the VMX transient structure.
441	*
442	* @returns VBox status code.
443	* @param pVCpu Pointer to the VMCPU.
444	* @param pVmxTransient Pointer to the VMX transient structure.
445	*
446	* @remarks No-long-jump zone!!!
447	*/
448	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
449	{
450	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
451	AssertRCReturn(rc, rc);
452	return VINF_SUCCESS;
453	}
454
455
456	/**
457	* Reads the VM-exit interruption-information field from the VMCS into the VMX
458	* transient structure.
459	*
460	* @returns VBox status code.
461	* @param pVCpu Pointer to the VMCPU.
462	* @param pVmxTransient Pointer to the VMX transient structure.
463	*/
464	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
465	{
466	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
467	{
468	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
469	AssertRCReturn(rc, rc);
470	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
471	}
472	return VINF_SUCCESS;
473	}
474
475
476	/**
477	* Reads the VM-exit interruption error code from the VMCS into the VMX
478	* transient structure.
479	*
480	* @returns VBox status code.
481	* @param pVCpu Pointer to the VMCPU.
482	* @param pVmxTransient Pointer to the VMX transient structure.
483	*/
484	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
485	{
486	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
487	{
488	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
489	AssertRCReturn(rc, rc);
490	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
491	}
492	return VINF_SUCCESS;
493	}
494
495
496	/**
497	* Reads the VM-exit instruction length field from the VMCS into the VMX
498	* transient structure.
499	*
500	* @returns VBox status code.
501	* @param pVCpu Pointer to the VMCPU.
502	* @param pVmxTransient Pointer to the VMX transient structure.
503	*/
504	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
505	{
506	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
507	{
508	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
509	AssertRCReturn(rc, rc);
510	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
511	}
512	return VINF_SUCCESS;
513	}
514
515
516	/**
517	* Reads the exit qualification from the VMCS into the VMX transient structure.
518	*
519	* @returns VBox status code.
520	* @param pVCpu Pointer to the VMCPU.
521	* @param pVmxTransient Pointer to the VMX transient structure.
522	*/
523	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
524	{
525	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
526	{
527	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
528	AssertRCReturn(rc, rc);
529	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
530	}
531	return VINF_SUCCESS;
532	}
533
534
535	/**
536	* Reads the IDT-vectoring information field from the VMCS into the VMX
537	* transient structure.
538	*
539	* @returns VBox status code.
540	* @param pVmxTransient Pointer to the VMX transient structure.
541	*
542	* @remarks No-long-jump zone!!!
543	*/
544	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
545	{
546	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
547	{
548	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
549	AssertRCReturn(rc, rc);
550	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
551	}
552	return VINF_SUCCESS;
553	}
554
555
556	/**
557	* Reads the IDT-vectoring error code from the VMCS into the VMX
558	* transient structure.
559	*
560	* @returns VBox status code.
561	* @param pVmxTransient Pointer to the VMX transient structure.
562	*/
563	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
564	{
565	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
566	{
567	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
568	AssertRCReturn(rc, rc);
569	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
570	}
571	return VINF_SUCCESS;
572	}
573
574
575	/**
576	* Enters VMX root mode operation on the current CPU.
577	*
578	* @returns VBox status code.
579	* @param pVM Pointer to the VM (optional, can be NULL, after
580	* a resume).
581	* @param HCPhysCpuPage Physical address of the VMXON region.
582	* @param pvCpuPage Pointer to the VMXON region.
583	*/
584	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
585	{
586	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
587	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
588
589	if (pVM)
590	{
591	/* Write the VMCS revision dword to the VMXON region. */
592	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
593	}
594
595	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
596	RTCCUINTREG fFlags = ASMIntDisableFlags();
597
598	/* Enable the VMX bit in CR4. */
599	RTCCUINTREG uCr4 = ASMGetCR4();
600	if (!(uCr4 & X86_CR4_VMXE))
601	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
602
603	/* Enter VMXON root mode. */
604	int rc = VMXEnable(HCPhysCpuPage);
605
606	/* Restore interrupts. */
607	ASMSetFlags(fFlags);
608	return rc;
609	}
610
611
612	/**
613	* Exits VMX root mode operation on the current CPU.
614	*
615	* @returns VBox status code.
616	*/
617	static int hmR0VmxLeaveRootMode(void)
618	{
619	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
620	RTCCUINTREG fFlags = ASMIntDisableFlags();
621	int rc = VINF_SUCCESS;
622
623	/* If we're for some reason not in VMX root mode, then don't leave it. */
624	if (ASMGetCR4() & X86_CR4_VMXE)
625	{
626	/* Exit root mode using VMXOFF & clear the VMX bit in CR4 */
627	VMXDisable();
628	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
629	}
630	else
631	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
632
633	/* Restore interrupts. */
634	ASMSetFlags(fFlags);
635	return rc;
636	}
637
638
639	/**
640	* Allocates and maps one physically contiguous page. The allocated page is
641	* zero'd out. (Used by various VT-x structures).
642	*
643	* @returns IPRT status code.
644	* @param pMemObj Pointer to the ring-0 memory object.
645	* @param ppVirt Where to store the virtual address of the
646	* allocation.
647	* @param pPhys Where to store the physical address of the
648	* allocation.
649	*/
650	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
651	{
652	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
653	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
654	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
655
656	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
657	if (RT_FAILURE(rc))
658	return rc;
659	ppVirt = RTR0MemObjAddress(pMemObj);
660	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
661	ASMMemZero32(*ppVirt, PAGE_SIZE);
662	return VINF_SUCCESS;
663	}
664
665
666	/**
667	* Frees and unmaps an allocated physical page.
668	*
669	* @param pMemObj Pointer to the ring-0 memory object.
670	* @param ppVirt Where to re-initialize the virtual address of
671	* allocation as 0.
672	* @param pHCPhys Where to re-initialize the physical address of the
673	* allocation as 0.
674	*/
675	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
676	{
677	AssertPtr(pMemObj);
678	AssertPtr(ppVirt);
679	AssertPtr(pHCPhys);
680	if (*pMemObj != NIL_RTR0MEMOBJ)
681	{
682	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
683	AssertRC(rc);
684	*pMemObj = NIL_RTR0MEMOBJ;
685	*ppVirt = 0;
686	*pHCPhys = 0;
687	}
688	}
689
690
691	/**
692	* Worker function to free VT-x related structures.
693	*
694	* @returns IPRT status code.
695	* @param pVM Pointer to the VM.
696	*/
697	static void hmR0VmxStructsFree(PVM pVM)
698	{
699	for (VMCPUID i = 0; i < pVM->cCpus; i++)
700	{
701	PVMCPU pVCpu = &pVM->aCpus[i];
702	AssertPtr(pVCpu);
703
704	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
705	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
706
707	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
708	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
709
710	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
711	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
712	}
713
714	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
715	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
716	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
717	#endif
718	}
719
720
721	/**
722	* Worker function to allocate VT-x related VM structures.
723	*
724	* @returns IPRT status code.
725	* @param pVM Pointer to the VM.
726	*/
727	static int hmR0VmxStructsAlloc(PVM pVM)
728	{
729	/*
730	* Initialize members up-front so we can cleanup properly on allocation failure.
731	*/
732	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
733	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
734	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
735	pVM->hm.s.vmx.HCPhys##a_Name = 0;
736
737	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
738	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
739	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
740	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
741
742	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
743	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
744	#endif
745	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
746
747	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
748	for (VMCPUID i = 0; i < pVM->cCpus; i++)
749	{
750	PVMCPU pVCpu = &pVM->aCpus[i];
751	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
752	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
753	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
754	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
755	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
756	}
757	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
758	#undef VMXLOCAL_INIT_VM_MEMOBJ
759
760	/*
761	* Allocate all the VT-x structures.
762	*/
763	int rc = VINF_SUCCESS;
764	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
765	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
766	if (RT_FAILURE(rc))
767	goto cleanup;
768	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
769	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
770	#endif
771
772	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
773	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
774	{
775	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
776	&pVM->hm.s.vmx.HCPhysApicAccess);
777	if (RT_FAILURE(rc))
778	goto cleanup;
779	}
780
781	/*
782	* Initialize per-VCPU VT-x structures.
783	*/
784	for (VMCPUID i =0; i < pVM->cCpus; i++)
785	{
786	PVMCPU pVCpu = &pVM->aCpus[i];
787	AssertPtr(pVCpu);
788
789	/* Allocate the VM control structure (VMCS). */
790	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
791	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
792	if (RT_FAILURE(rc))
793	goto cleanup;
794
795	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
796	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
797	{
798	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
799	&pVCpu->hm.s.vmx.HCPhysVirtApic);
800	if (RT_FAILURE(rc))
801	goto cleanup;
802	}
803
804	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
805	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
806	{
807	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
808	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
809	if (RT_FAILURE(rc))
810	goto cleanup;
811	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
812	}
813
814	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
815	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
816	if (RT_FAILURE(rc))
817	goto cleanup;
818
819	/* Allocate the VM-exit MSR-load page for the host MSRs. */
820	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
821	if (RT_FAILURE(rc))
822	goto cleanup;
823	}
824
825	return VINF_SUCCESS;
826
827	cleanup:
828	hmR0VmxStructsFree(pVM);
829	return rc;
830	}
831
832
833	/**
834	* Does global VT-x initialization (called during module initialization).
835	*
836	* @returns VBox status code.
837	*/
838	VMMR0DECL(int) VMXR0GlobalInit(void)
839	{
840	/* Setup the main VM exit handlers. */
841	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(s_apfnVMExitHandlers));
842	#ifdef DEBUG
843	for (unsigned i = 0; i < RT_ELEMENTS(s_apfnVMExitHandlers); i++)
844	Assert(s_apfnVMExitHandlers[i]);
845	#endif
846	return VINF_SUCCESS;
847	}
848
849
850	/**
851	* Does global VT-x termination (called during module termination).
852	*/
853	VMMR0DECL(void) VMXR0GlobalTerm()
854	{
855	/* Nothing to do currently. */
856	}
857
858
859	/**
860	* Sets up and activates VT-x on the current CPU.
861	*
862	* @returns VBox status code.
863	* @param pCpu Pointer to the global CPU info struct.
864	* @param pVM Pointer to the VM (can be NULL after a host resume
865	* operation).
866	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
867	* fEnabledByHost is true).
868	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
869	* @a fEnabledByHost is true).
870	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
871	* enable VT-x/AMD-V on the host.
872	*/
873	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
874	{
875	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
876	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
877
878	if (!fEnabledByHost)
879	{
880	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
881	if (RT_FAILURE(rc))
882	return rc;
883	}
884
885	/*
886	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
887	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
888	* each time while reusing a VPID after hitting the MaxASID limit once.
889	*/
890	if ( pVM
891	&& pVM->hm.s.vmx.fVpid
892	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
893	{
894	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
895	pCpu->fFlushAsidBeforeUse = false;
896	}
897	else
898	pCpu->fFlushAsidBeforeUse = true;
899
900	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
901	++pCpu->cTlbFlushes;
902
903	return VINF_SUCCESS;
904	}
905
906
907	/**
908	* Deactivates VT-x on the current CPU.
909	*
910	* @returns VBox status code.
911	* @param pCpu Pointer to the global CPU info struct.
912	* @param pvCpuPage Pointer to the VMXON region.
913	* @param HCPhysCpuPage Physical address of the VMXON region.
914	*/
915	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
916	{
917	NOREF(pCpu);
918	NOREF(pvCpuPage);
919	NOREF(HCPhysCpuPage);
920
921	hmR0VmxLeaveRootMode();
922	return VINF_SUCCESS;
923	}
924
925
926	/**
927	* Sets the permission bits for the specified MSR in the MSR bitmap.
928	*
929	* @param pVCpu Pointer to the VMCPU.
930	* @param ulMSR The MSR value.
931	* @param enmRead Whether reading this MSR causes a VM-exit.
932	* @param enmWrite Whether writing this MSR causes a VM-exit.
933	*/
934	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, unsigned ulMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
935	{
936	unsigned ulBit;
937	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
938
939	/*
940	* Layout:
941	* 0x000 - 0x3ff - Low MSR read bits
942	* 0x400 - 0x7ff - High MSR read bits
943	* 0x800 - 0xbff - Low MSR write bits
944	* 0xc00 - 0xfff - High MSR write bits
945	*/
946	if (ulMsr <= 0x00001FFF)
947	{
948	/* Pentium-compatible MSRs */
949	ulBit = ulMsr;
950	}
951	else if ( ulMsr >= 0xC0000000
952	&& ulMsr <= 0xC0001FFF)
953	{
954	/* AMD Sixth Generation x86 Processor MSRs */
955	ulBit = (ulMsr - 0xC0000000);
956	pbMsrBitmap += 0x400;
957	}
958	else
959	{
960	AssertMsgFailed(("Invalid MSR %lx\n", ulMsr));
961	return;
962	}
963
964	Assert(ulBit <= 0x1fff);
965	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
966	ASMBitSet(pbMsrBitmap, ulBit);
967	else
968	ASMBitClear(pbMsrBitmap, ulBit);
969
970	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
971	ASMBitSet(pbMsrBitmap + 0x800, ulBit);
972	else
973	ASMBitClear(pbMsrBitmap + 0x800, ulBit);
974	}
975
976
977	/**
978	* Flushes the TLB using EPT.
979	*
980	* @returns VBox status code.
981	* @param pVM Pointer to the VM.
982	* @param pVCpu Pointer to the VMCPU.
983	* @param enmFlush Type of flush.
984	*/
985	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
986	{
987	AssertPtr(pVM);
988	Assert(pVM->hm.s.fNestedPaging);
989
990	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
991
992	uint64_t descriptor[2];
993	descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
994	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
995
996	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
997	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
998	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
999	}
1000
1001
1002	/**
1003	* Flushes the TLB using VPID.
1004	*
1005	* @returns VBox status code.
1006	* @param pVM Pointer to the VM.
1007	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1008	* enmFlush).
1009	* @param enmFlush Type of flush.
1010	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1011	* on @a enmFlush).
1012	*/
1013	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1014	{
1015	AssertPtr(pVM);
1016	Assert(pVM->hm.s.vmx.fVpid);
1017
1018	uint64_t descriptor[2];
1019	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1020	{
1021	descriptor[0] = 0;
1022	descriptor[1] = 0;
1023	}
1024	else
1025	{
1026	AssertPtr(pVCpu);
1027	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1028	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1029	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1030	descriptor[1] = GCPtr;
1031	}
1032
1033	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1034	AssertMsg(rc == VINF_SUCCESS,
1035	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1036	if ( RT_SUCCESS(rc)
1037	&& pVCpu)
1038	{
1039	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1040	}
1041	}
1042
1043
1044	/**
1045	* Invalidates a guest page by guest virtual address. Only relevant for
1046	* EPT/VPID, otherwise there is nothing really to invalidate.
1047	*
1048	* @returns VBox status code.
1049	* @param pVM Pointer to the VM.
1050	* @param pVCpu Pointer to the VMCPU.
1051	* @param GCVirt Guest virtual address of the page to invalidate.
1052	*/
1053	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1054	{
1055	AssertPtr(pVM);
1056	AssertPtr(pVCpu);
1057	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1058
1059	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1060	if (!fFlushPending)
1061	{
1062	/*
1063	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1064	* See @bugref{6043} and @bugref{6177}.
1065	*
1066	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1067	* function maybe called in a loop with individual addresses.
1068	*/
1069	if (pVM->hm.s.vmx.fVpid)
1070	{
1071	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1072	{
1073	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1074	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1075	}
1076	else
1077	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1078	}
1079	else if (pVM->hm.s.fNestedPaging)
1080	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1081	}
1082
1083	return VINF_SUCCESS;
1084	}
1085
1086
1087	/**
1088	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1089	* otherwise there is nothing really to invalidate.
1090	*
1091	* @returns VBox status code.
1092	* @param pVM Pointer to the VM.
1093	* @param pVCpu Pointer to the VMCPU.
1094	* @param GCPhys Guest physical address of the page to invalidate.
1095	*/
1096	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1097	{
1098	LogFlowFunc(("%RGp\n", GCPhys));
1099
1100	/*
1101	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1102	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1103	* This function might be called in a loop.
1104	*/
1105	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1106	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1107	return VINF_SUCCESS;
1108	}
1109
1110
1111	/**
1112	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1113	* case where neither EPT nor VPID is supported by the CPU.
1114	*
1115	* @param pVM Pointer to the VM.
1116	* @param pVCpu Pointer to the VMCPU.
1117	*
1118	* @remarks Called with interrupts disabled.
1119	*/
1120	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1121	{
1122	NOREF(pVM);
1123	AssertPtr(pVCpu);
1124	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1125	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1126
1127	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1128	AssertPtr(pCpu);
1129
1130	pVCpu->hm.s.TlbShootdown.cPages = 0;
1131	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1132	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1133	pVCpu->hm.s.fForceTLBFlush = false;
1134	return;
1135	}
1136
1137
1138	/**
1139	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1140	*
1141	* @param pVM Pointer to the VM.
1142	* @param pVCpu Pointer to the VMCPU.
1143	* @remarks All references to "ASID" in this function pertains to "VPID" in
1144	* Intel's nomenclature. The reason is, to avoid confusion in compare
1145	* statements since the host-CPU copies are named "ASID".
1146	*
1147	* @remarks Called with interrupts disabled.
1148	*/
1149	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1150	{
1151	AssertPtr(pVM);
1152	AssertPtr(pVCpu);
1153	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1154	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1155	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1156
1157	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1158	AssertPtr(pCpu);
1159
1160	/*
1161	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1162	* This can happen both for start & resume due to long jumps back to ring-3.
1163	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1164	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1165	*/
1166	bool fNewASID = false;
1167	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1168	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1169	{
1170	pVCpu->hm.s.fForceTLBFlush = true;
1171	fNewASID = true;
1172	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1173	}
1174
1175	/*
1176	* Check for explicit TLB shootdowns.
1177	*/
1178	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1179	{
1180	pVCpu->hm.s.fForceTLBFlush = true;
1181	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1182	}
1183
1184	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1185	if (pVCpu->hm.s.fForceTLBFlush)
1186	{
1187	if (fNewASID)
1188	{
1189	++pCpu->uCurrentAsid;
1190	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1191	{
1192	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1193	pCpu->cTlbFlushes++;
1194	pCpu->fFlushAsidBeforeUse = true;
1195	}
1196
1197	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1198	if (pCpu->fFlushAsidBeforeUse)
1199	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1200	}
1201	else
1202	{
1203	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1204	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1205	else
1206	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1207	}
1208
1209	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1210	pVCpu->hm.s.fForceTLBFlush = false;
1211	}
1212	else
1213	{
1214	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1215	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1216	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1217	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1218
1219	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1220	* not be executed. See hmQueueInvlPage() where it is commented
1221	* out. Support individual entry flushing someday. */
1222	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1223	{
1224	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1225
1226	/*
1227	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1228	* as supported by the CPU.
1229	*/
1230	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1231	{
1232	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1233	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1234	}
1235	else
1236	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1237	}
1238	else
1239	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1240	}
1241	pVCpu->hm.s.TlbShootdown.cPages = 0;
1242	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1243
1244	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1245	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1246	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1247	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1248	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1249	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1250
1251	/* Update VMCS with the VPID. */
1252	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1253	AssertRC(rc);
1254	}
1255
1256
1257	/**
1258	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1259	*
1260	* @returns VBox status code.
1261	* @param pVM Pointer to the VM.
1262	* @param pVCpu Pointer to the VMCPU.
1263	*
1264	* @remarks Called with interrupts disabled.
1265	*/
1266	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1267	{
1268	AssertPtr(pVM);
1269	AssertPtr(pVCpu);
1270	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1271	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1272
1273	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1274	AssertPtr(pCpu);
1275
1276	/*
1277	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1278	* This can happen both for start & resume due to long jumps back to ring-3.
1279	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1280	*/
1281	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1282	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1283	{
1284	pVCpu->hm.s.fForceTLBFlush = true;
1285	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1286	}
1287
1288	/* Check for explicit TLB shootdown flushes. */
1289	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1290	{
1291	pVCpu->hm.s.fForceTLBFlush = true;
1292	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1293	}
1294
1295	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1296	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1297
1298	if (pVCpu->hm.s.fForceTLBFlush)
1299	{
1300	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1301	pVCpu->hm.s.fForceTLBFlush = false;
1302	}
1303	else
1304	{
1305	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1306	* not be executed. See hmQueueInvlPage() where it is commented
1307	* out. Support individual entry flushing someday. */
1308	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1309	{
1310	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1311	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1312	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1313	}
1314	else
1315	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1316	}
1317
1318	pVCpu->hm.s.TlbShootdown.cPages= 0;
1319	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1320	}
1321
1322
1323	/**
1324	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1325	*
1326	* @returns VBox status code.
1327	* @param pVM Pointer to the VM.
1328	* @param pVCpu Pointer to the VMCPU.
1329	*
1330	* @remarks Called with interrupts disabled.
1331	*/
1332	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1333	{
1334	AssertPtr(pVM);
1335	AssertPtr(pVCpu);
1336	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1337	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1338
1339	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1340
1341	/*
1342	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1343	* This can happen both for start & resume due to long jumps back to ring-3.
1344	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1345	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1346	*/
1347	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1348	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1349	{
1350	pVCpu->hm.s.fForceTLBFlush = true;
1351	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1352	}
1353
1354	/* Check for explicit TLB shootdown flushes. */
1355	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1356	{
1357	/*
1358	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1359	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1360	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1361	*/
1362	pVCpu->hm.s.fForceTLBFlush = true;
1363	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1364	}
1365
1366	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1367	if (pVCpu->hm.s.fForceTLBFlush)
1368	{
1369	++pCpu->uCurrentAsid;
1370	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1371	{
1372	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1373	pCpu->fFlushAsidBeforeUse = true;
1374	pCpu->cTlbFlushes++;
1375	}
1376
1377	pVCpu->hm.s.fForceTLBFlush = false;
1378	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1379	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1380	if (pCpu->fFlushAsidBeforeUse)
1381	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1382	}
1383	else
1384	{
1385	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1386	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1387	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1388	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1389
1390	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1391	* not be executed. See hmQueueInvlPage() where it is commented
1392	* out. Support individual entry flushing someday. */
1393	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1394	{
1395	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1396	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1397	{
1398	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1399	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1400	}
1401	else
1402	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1403	}
1404	else
1405	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1406	}
1407
1408	pVCpu->hm.s.TlbShootdown.cPages = 0;
1409	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1410
1411	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1412	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1413	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1414	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1415	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1416	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1417
1418	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1419	AssertRC(rc);
1420	}
1421
1422
1423	/**
1424	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1425	* TLB entries from the host TLB before VM-entry.
1426	*
1427	* @returns VBox status code.
1428	* @param pVM Pointer to the VM.
1429	*/
1430	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1431	{
1432	/*
1433	* Determine optimal flush type for nested paging.
1434	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1435	* guest execution (see hmR3InitFinalizeR0()).
1436	*/
1437	if (pVM->hm.s.fNestedPaging)
1438	{
1439	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1440	{
1441	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1442	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1443	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1444	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1445	else
1446	{
1447	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1448	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1449	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1450	}
1451
1452	/* Make sure the write-back cacheable memory type for EPT is supported. */
1453	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1454	{
1455	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1456	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1457	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1458	}
1459	}
1460	else
1461	{
1462	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1463	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1464	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1465	}
1466	}
1467
1468	/*
1469	* Determine optimal flush type for VPID.
1470	*/
1471	if (pVM->hm.s.vmx.fVpid)
1472	{
1473	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1474	{
1475	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1476	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1477	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1478	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1479	else
1480	{
1481	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1482	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1483	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1484	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1485	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1486	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1487	pVM->hm.s.vmx.fVpid = false;
1488	}
1489	}
1490	else
1491	{
1492	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1493	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1494	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1495	pVM->hm.s.vmx.fVpid = false;
1496	}
1497	}
1498
1499	/*
1500	* Setup the handler for flushing tagged-TLBs.
1501	*/
1502	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1503	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbBoth;
1504	else if (pVM->hm.s.fNestedPaging)
1505	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbEpt;
1506	else if (pVM->hm.s.vmx.fVpid)
1507	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbVpid;
1508	else
1509	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbNone;
1510	return VINF_SUCCESS;
1511	}
1512
1513
1514	/**
1515	* Sets up pin-based VM-execution controls in the VMCS.
1516	*
1517	* @returns VBox status code.
1518	* @param pVM Pointer to the VM.
1519	* @param pVCpu Pointer to the VMCPU.
1520	*/
1521	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1522	{
1523	AssertPtr(pVM);
1524	AssertPtr(pVCpu);
1525
1526	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1527	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1528
1529	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1530	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1531	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1532
1533	/* Enable the VMX preemption timer. */
1534	if (pVM->hm.s.vmx.fUsePreemptTimer)
1535	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1536
1537	if ((val & zap) != val)
1538	{
1539	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1540	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1541	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1542	}
1543
1544	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1545	AssertRCReturn(rc, rc);
1546
1547	/* Update VCPU with the currently set pin-based VM-execution controls. */
1548	pVCpu->hm.s.vmx.u32PinCtls = val;
1549	return rc;
1550	}
1551
1552
1553	/**
1554	* Sets up processor-based VM-execution controls in the VMCS.
1555	*
1556	* @returns VBox status code.
1557	* @param pVM Pointer to the VM.
1558	* @param pVMCPU Pointer to the VMCPU.
1559	*/
1560	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1561	{
1562	AssertPtr(pVM);
1563	AssertPtr(pVCpu);
1564
1565	int rc = VERR_INTERNAL_ERROR_5;
1566	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1567	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1568
1569	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1570	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1571	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1572	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1573	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1574	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1575	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1576
1577	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1578	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1579	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1580	{
1581	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1582	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1583	}
1584
1585	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1586	if (!pVM->hm.s.fNestedPaging)
1587	{
1588	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1589	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1590	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1591	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1592	}
1593
1594	/* Use TPR shadowing if supported by the CPU. */
1595	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1596	{
1597	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1598	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1599	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1600	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1601	AssertRCReturn(rc, rc);
1602
1603	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1604	/* CR8 writes causes a VM-exit based on TPR threshold. */
1605	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1606	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1607	}
1608	else
1609	{
1610	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1611	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1612	}
1613
1614	/* Use MSR-bitmaps if supported by the CPU. */
1615	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1616	{
1617	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1618
1619	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1620	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1621	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1622	AssertRCReturn(rc, rc);
1623
1624	/*
1625	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1626	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1627	*/
1628	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1629	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1630	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1631	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1632	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1633	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1634	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1635	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1636	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1637	}
1638
1639	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1640	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1641	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1642
1643	if ((val & zap) != val)
1644	{
1645	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1646	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1647	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1648	}
1649
1650	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1651	AssertRCReturn(rc, rc);
1652
1653	/* Update VCPU with the currently set processor-based VM-execution controls. */
1654	pVCpu->hm.s.vmx.u32ProcCtls = val;
1655
1656	/*
1657	* Secondary processor-based VM-execution controls.
1658	*/
1659	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1660	{
1661	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1662	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1663
1664	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1665	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1666
1667	if (pVM->hm.s.fNestedPaging)
1668	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1669	else
1670	{
1671	/*
1672	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1673	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1674	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1675	*/
1676	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1677	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1678	}
1679
1680	if (pVM->hm.s.vmx.fVpid)
1681	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1682
1683	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1684	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1685
1686	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1687	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1688	* done dynamically. */
1689	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1690	{
1691	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1692	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1693	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1694	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1695	AssertRCReturn(rc, rc);
1696	}
1697
1698	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1699	{
1700	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1701	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1702	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1703	}
1704
1705	if ((val & zap) != val)
1706	{
1707	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1708	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1709	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1710	}
1711
1712	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1713	AssertRCReturn(rc, rc);
1714
1715	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1716	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1717	}
1718
1719	return VINF_SUCCESS;
1720	}
1721
1722
1723	/**
1724	* Sets up miscellaneous (everything other than Pin & Processor-based
1725	* VM-execution) control fields in the VMCS.
1726	*
1727	* @returns VBox status code.
1728	* @param pVM Pointer to the VM.
1729	* @param pVCpu Pointer to the VMCPU.
1730	*/
1731	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1732	{
1733	AssertPtr(pVM);
1734	AssertPtr(pVCpu);
1735
1736	int rc = VERR_GENERAL_FAILURE;
1737
1738	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1739	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1740
1741	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1742
1743	/*
1744	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1745	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1746	* We thus use the exception bitmap to control it rather than use both.
1747	*/
1748	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1749	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1750
1751	/** @todo Explore possibility of using IO-bitmaps. */
1752	/* All IO & IOIO instructions cause VM-exits. */
1753	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1754	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1755
1756	/* Setup MSR autoloading/autostoring. */
1757	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1758	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1759	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1760	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1761	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1762	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1763
1764	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1765	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1766	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1767	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1768
1769	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1770	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1771
1772	/* Setup debug controls */
1773	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1774	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1775	AssertRCReturn(rc, rc);
1776	return rc;
1777	}
1778
1779
1780	/**
1781	* Sets up the initial exception bitmap in the VMCS based on static conditions
1782	* (i.e. conditions that cannot ever change at runtime).
1783	*
1784	* @returns VBox status code.
1785	* @param pVM Pointer to the VM.
1786	* @param pVCpu Pointer to the VMCPU.
1787	*/
1788	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1789	{
1790	AssertPtr(pVM);
1791	AssertPtr(pVCpu);
1792
1793	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1794
1795	uint32_t u32XcptBitmap = 0;
1796
1797	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1798	if (!pVM->hm.s.fNestedPaging)
1799	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1800
1801	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1802	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1803	AssertRCReturn(rc, rc);
1804	return rc;
1805	}
1806
1807
1808	/**
1809	* Sets up the initial guest-state mask. The guest-state mask is consulted
1810	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1811	* for the nested virtualization case (as it would cause a VM-exit).
1812	*
1813	* @param pVCpu Pointer to the VMCPU.
1814	*/
1815	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1816	{
1817	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1818	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1819	return VINF_SUCCESS;
1820	}
1821
1822
1823	/**
1824	* Does per-VM VT-x initialization.
1825	*
1826	* @returns VBox status code.
1827	* @param pVM Pointer to the VM.
1828	*/
1829	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1830	{
1831	LogFlowFunc(("pVM=%p\n", pVM));
1832
1833	int rc = hmR0VmxStructsAlloc(pVM);
1834	if (RT_FAILURE(rc))
1835	{
1836	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1837	return rc;
1838	}
1839
1840	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1841	{
1842	PVMCPU pVCpu = &pVM->aCpus[i];
1843
1844	/* Current guest paging mode. */
1845	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1846	}
1847
1848	return VINF_SUCCESS;
1849	}
1850
1851
1852	/**
1853	* Does per-VM VT-x termination.
1854	*
1855	* @returns VBox status code.
1856	* @param pVM Pointer to the VM.
1857	*/
1858	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1859	{
1860	LogFlowFunc(("pVM=%p\n", pVM));
1861
1862	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1863	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1864	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1865	#endif
1866	hmR0VmxStructsFree(pVM);
1867	return VINF_SUCCESS;
1868	}
1869
1870
1871	/**
1872	* Sets up the VM for execution under VT-x.
1873	* This function is only called once per-VM during initalization.
1874	*
1875	* @returns VBox status code.
1876	* @param pVM Pointer to the VM.
1877	*/
1878	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1879	{
1880	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1881	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1882
1883	LogFlowFunc(("pVM=%p\n", pVM));
1884
1885	/*
1886	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1887	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1888	*/
1889	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1890	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1891	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1892	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1893	{
1894	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1895	return VERR_INTERNAL_ERROR;
1896	}
1897
1898	/* Initialize these always, see hmR3InitFinalizeR0().*/
1899	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1900	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1901
1902	/* Setup the tagged-TLB flush handlers. */
1903	int rc = hmR0VmxSetupTaggedTlb(pVM);
1904	if (RT_FAILURE(rc))
1905	{
1906	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1907	return rc;
1908	}
1909
1910	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1911	{
1912	PVMCPU pVCpu = &pVM->aCpus[i];
1913	AssertPtr(pVCpu);
1914	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1915
1916	/* Set revision dword at the beginning of the VMCS structure. */
1917	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1918
1919	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1920	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1921	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1922	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1923
1924	/* Load this VMCS as the current VMCS. */
1925	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1926	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1927	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1928
1929	/* Setup the pin-based VM-execution controls. */
1930	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1931	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1932	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1933
1934	/* Setup the processor-based VM-execution controls. */
1935	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1936	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1937	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1938
1939	/* Setup the rest (miscellaneous) VM-execution controls. */
1940	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1941	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1942	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1943
1944	/* Setup the initial exception bitmap. */
1945	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1946	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1947	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1948
1949	/* Setup the initial guest-state mask. */
1950	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1951	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1952	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1953
1954	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1955	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1956	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1957	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1958	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1959	#endif
1960
1961	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
1962	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1963	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1964	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1965
1966	/* Update the last error record for this VCPU. */
1967	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
1968	}
1969
1970	return VINF_SUCCESS;
1971	}
1972
1973
1974	/**
1975	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
1976	* the VMCS.
1977	*
1978	* @returns VBox status code.
1979	* @param pVM Pointer to the VM.
1980	* @param pVCpu Pointer to the VMCPU.
1981	*/
1982	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
1983	{
1984	RTCCUINTREG uReg = ASMGetCR0();
1985	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
1986
1987	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1988	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
1989	if (VMX_IS_64BIT_HOST_MODE())
1990	{
1991	uint64_t uReg = hmR0Get64bitCR3();
1992	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
1993	}
1994	else
1995	#endif
1996	{
1997	uReg = ASMGetCR3();
1998	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
1999	}
2000
2001	uReg = ASMGetCR4();
2002	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2003	AssertRCReturn(rc, rc);
2004	return rc;
2005	}
2006
2007
2008	/**
2009	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2010	* the host-state area in the VMCS.
2011	*
2012	* @returns VBox status code.
2013	* @param pVM Pointer to the VM.
2014	* @param pVCpu Pointer to the VMCPU.
2015	*/
2016	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2017	{
2018	int rc = VERR_INTERNAL_ERROR_5;
2019	RTSEL uSelCS = 0;
2020	RTSEL uSelSS = 0;
2021	RTSEL uSelDS = 0;
2022	RTSEL uSelES = 0;
2023	RTSEL uSelFS = 0;
2024	RTSEL uSelGS = 0;
2025	RTSEL uSelTR = 0;
2026
2027	/*
2028	* Host Selector registers.
2029	*/
2030	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2031	if (VMX_IS_64BIT_HOST_MODE())
2032	{
2033	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2034	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2035	}
2036	else
2037	{
2038	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2039	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2040	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2041	}
2042	#else
2043	uSelCS = ASMGetCS();
2044	uSelSS = ASMGetSS();
2045	#endif
2046
2047	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2048	/** @todo Verify if we have any platform that actually run with DS or ES with
2049	* RPL != 0 in kernel space. */
2050	uSelDS = 0;
2051	uSelES = 0;
2052	uSelFS = 0;
2053	uSelGS = 0;
2054	uSelTR = ASMGetTR();
2055
2056	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2057	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2058	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2059	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2060	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2061	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2062	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2063	Assert(uSelCS != 0);
2064	Assert(uSelTR != 0);
2065
2066	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2067	#if 0
2068	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2069	Assert(uSelSS != 0);
2070	#endif
2071
2072	/* Write these host selector fields into the host-state area in the VMCS. */
2073	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2074	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2075	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2076	#if 0
2077	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2078	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2079	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2080	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2081	#endif
2082	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2083	AssertRCReturn(rc, rc);
2084
2085	/*
2086	* Host GDTR and IDTR.
2087	*/
2088	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2089	* be safe to -not- save and restore GDTR and IDTR in the assembly
2090	* code and just do it here and don't care if the limits are zapped on
2091	* VM-exit. */
2092	RTGDTR Gdtr;
2093	RT_ZERO(Gdtr);
2094	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2095	if (VMX_IS_64BIT_HOST_MODE())
2096	{
2097	X86XDTR64 Gtr64;
2098	X86XDTR64 Idtr64;
2099	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2100	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2101	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2102	Gdtr.cbGdt = Gdtr64.cb;
2103	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2104	}
2105	else
2106	#endif
2107	{
2108	RTIDTR Idtr;
2109	ASMGetGDTR(&Gdtr);
2110	ASMGetIDTR(&Idtr);
2111	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2112	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2113	}
2114	AssertRCReturn(rc, rc);
2115
2116	/*
2117	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2118	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2119	*/
2120	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2121	{
2122	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2123	return VERR_VMX_INVALID_HOST_STATE;
2124	}
2125
2126	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2127	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2128	if (VMX_IS_64BIT_HOST_MODE())
2129	{
2130	/* We need the 64-bit TR base for hybrid darwin. */
2131	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2132	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2133	}
2134	else
2135	#endif
2136	{
2137	uintptr_t uTRBase;
2138	#if HC_ARCH_BITS == 64
2139	uTRBase = X86DESC64_BASE(pDesc);
2140	#else
2141	uTRBase = X86DESC_BASE(pDesc);
2142	#endif
2143	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2144	}
2145	AssertRCReturn(rc, rc);
2146
2147	/*
2148	* Host FS base and GS base.
2149	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2150	* would take care of the bases. In 64-bit, the MSRs come into play.
2151	*/
2152	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2153	if (VMX_IS_64BIT_HOST_MODE())
2154	{
2155	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2156	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2157	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2158	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2159	AssertRCReturn(rc, rc);
2160	}
2161	#endif
2162	return rc;
2163	}
2164
2165
2166	/**
2167	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2168	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2169	* the host after every successful VM exit.
2170	*
2171	* @returns VBox status code.
2172	* @param pVM Pointer to the VM.
2173	* @param pVCpu Pointer to the VMCPU.
2174	*/
2175	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2176	{
2177	AssertPtr(pVCpu);
2178	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2179
2180	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2181	unsigned idxHostMsr = 0;
2182	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2183
2184	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2185	{
2186	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2187	pHostMsr->u32Reserved = 0;
2188	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2189	if (CPUMIsGuestInLongMode(pVCpu))
2190	{
2191	/* Must match the EFER value in our 64 bits switcher. */
2192	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2193	}
2194	else
2195	#endif
2196	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2197	pHostMsr++; idxHostMsr++;
2198	}
2199
2200	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2201	if (VMX_IS_64BIT_HOST_MODE())
2202	{
2203	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2204	pHostMsr->u32Reserved = 0;
2205	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2206	pHostMsr++; idxHostMsr++;
2207	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2208	pHostMsr->u32Reserved = 0;
2209	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2210	pHostMsr++; idxHostMsr++;
2211	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2212	pHostMsr->u32Reserved = 0;
2213	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2214	pHostMsr++; idxHostMsr++;
2215	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2216	#if 0
2217	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2218	pMsr->u32Reserved = 0;
2219	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2220	pHostMsr++; idxHostMsr++;
2221	#endif
2222	}
2223	#endif
2224
2225	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2226	if (RT_UNLIKELY(idxHostMsr > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2227	{
2228	LogRel(("idxHostMsr=%u Cpu=%u\n", idxHostMsr, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2229	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2230	}
2231
2232	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxHostMsr);
2233
2234	/*
2235	* Host Sysenter MSRs.
2236	*/
2237	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2238	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2239	if (VMX_IS_64BIT_HOST_MODE())
2240	{
2241	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2242	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2243	}
2244	else
2245	{
2246	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2247	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2248	}
2249	#elif HC_ARCH_BITS == 32
2250	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2251	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2252	#else
2253	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2254	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2255	#endif
2256	AssertRCReturn(rc, rc);
2257
2258	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2259	* hmR0VmxSetupExitCtls() !! */
2260	return rc;
2261	}
2262
2263
2264	/**
2265	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2266	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2267	* controls".
2268	*
2269	* @returns VBox status code.
2270	* @param pVM Pointer to the VM.
2271	* @param pVCpu Pointer to the VMCPU.
2272	* @param pCtx Pointer to the guest-CPU context.
2273	*
2274	* @remarks No-long-jump zone!!!
2275	*/
2276	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2277	{
2278	int rc = VINF_SUCCESS;
2279	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2280	{
2281	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2282	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2283
2284	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2285	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2286
2287	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2288	if (CPUMIsGuestInLongModeEx(pCtx))
2289	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2290	else
2291	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2292
2293	/*
2294	* The following should not be set (since we're not in SMM mode):
2295	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2296	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2297	*/
2298
2299	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2300	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2301	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2302
2303	if ((val & zap) != val)
2304	{
2305	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2306	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2307	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2308	}
2309
2310	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2311	AssertRCReturn(rc, rc);
2312
2313	/* Update VCPU with the currently set VM-exit controls. */
2314	pVCpu->hm.s.vmx.u32EntryCtls = val;
2315	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2316	}
2317	return rc;
2318	}
2319
2320
2321	/**
2322	* Sets up the VM-exit controls in the VMCS.
2323	*
2324	* @returns VBox status code.
2325	* @param pVM Pointer to the VM.
2326	* @param pVCpu Pointer to the VMCPU.
2327	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2328	* out-of-sync. Make sure to update the required fields
2329	* before using them.
2330	*
2331	* @remarks requires EFER.
2332	*/
2333	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2334	{
2335	int rc = VINF_SUCCESS;
2336	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2337	{
2338	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2339	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2340
2341	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2342	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2343
2344	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2345	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2346	if (VMX_IS_64BIT_HOST_MODE())
2347	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2348	else
2349	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2350	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2351	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2352	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2353	else
2354	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2355	#endif
2356
2357	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2358	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2359
2360	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2361	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2362	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2363	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2364	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2365
2366	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2367	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2368
2369	if ((val & zap) != val)
2370	{
2371	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2372	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2373	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2374	}
2375
2376	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2377	AssertRCReturn(rc, rc);
2378
2379	/* Update VCPU with the currently set VM-exit controls. */
2380	pVCpu->hm.s.vmx.u32ExitCtls = val;
2381	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2382	}
2383	return rc;
2384	}
2385
2386
2387	/**
2388	* Loads the guest APIC and related state.
2389	*
2390	* @returns VBox status code.
2391	* @param pVM Pointer to the VM.
2392	* @param pVCpu Pointer to the VMCPU.
2393	* @param pCtx Pointer to the guest-CPU context.
2394	*/
2395	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2396	{
2397	int rc = VINF_SUCCESS;
2398	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2399	{
2400	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2401	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2402	{
2403	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2404
2405	bool fPendingIntr = false;
2406	uint8_t u8GuestTpr = 0;
2407	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2408	AssertRCReturn(rc, rc);
2409
2410	/*
2411	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2412	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2413	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2414	* the interrupt when we VM-exit for other reasons.
2415	*/
2416	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2417	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2418	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2419	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2420
2421	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2422	AssertRCReturn(rc, rc);
2423
2424	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2425	if (pVM->hm.s.fTPRPatchingActive)
2426	{
2427	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2428	pCtx->msrLSTAR = u8GuestTpr;
2429	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2430	{
2431	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2432	if (fPendingIntr)
2433	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2434	else
2435	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2436	}
2437	}
2438	}
2439
2440	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2441	}
2442	return rc;
2443	}
2444
2445
2446	/**
2447	* Loads the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
2448	* into the guest-state area in the VMCS.
2449	*
2450	* @param pVM Pointer to the VM.
2451	* @param pVCpu Pointer to the VMCPU.
2452	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2453	* out-of-sync. Make sure to update the required fields
2454	* before using them.
2455	*
2456	* @remarks No-long-jump zone!!!
2457	*/
2458	DECLINLINE(void) hmR0VmxLoadGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2459	{
2460	/*
2461	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2462	* inhibit interrupts or clear any existing interrupt-inhibition.
2463	*/
2464	uint32_t uIntrState = 0;
2465	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2466	{
2467	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2468	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS)),
2469	("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2470	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2471	{
2472	/*
2473	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2474	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2475	* hmR0VmxInjectPendingInterrupt() relies on us clearing this flag here.
2476	*/
2477	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2478	}
2479	else if (pMixedCtx->eflags.Bits.u1IF)
2480	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2481	else
2482	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2483	}
2484
2485	Assert(!(uIntrState & 0xfffffff0)); /* Bits 31:4 MBZ. */
2486	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2487	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2488	AssertRC(rc);
2489	}
2490
2491
2492	/**
2493	* Loads the guest's RIP into the guest-state area in the VMCS.
2494	*
2495	* @returns VBox status code.
2496	* @param pVM Pointer to the VM.
2497	* @param pVCpu Pointer to the VMCPU.
2498	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2499	* out-of-sync. Make sure to update the required fields
2500	* before using them.
2501	*
2502	* @remarks No-long-jump zone!!!
2503	*/
2504	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2505	{
2506	int rc = VINF_SUCCESS;
2507	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2508	{
2509	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2510	AssertRCReturn(rc, rc);
2511	Log(("VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2512	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2513	}
2514	return rc;
2515	}
2516
2517
2518	/**
2519	* Loads the guest's RSP into the guest-state area in the VMCS.
2520	*
2521	* @returns VBox status code.
2522	* @param pVM Pointer to the VM.
2523	* @param pVCpu Pointer to the VMCPU.
2524	* @param pCtx Pointer to the guest-CPU context.
2525	*
2526	* @remarks No-long-jump zone!!!
2527	*/
2528	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2529	{
2530	int rc = VINF_SUCCESS;
2531	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2532	{
2533	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2534	AssertRCReturn(rc, rc);
2535	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2536	}
2537	return rc;
2538	}
2539
2540
2541	/**
2542	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2543	*
2544	* @returns VBox status code.
2545	* @param pVM Pointer to the VM.
2546	* @param pVCpu Pointer to the VMCPU.
2547	* @param pCtx Pointer to the guest-CPU context.
2548	*
2549	* @remarks No-long-jump zone!!!
2550	*/
2551	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2552	{
2553	int rc = VINF_SUCCESS;
2554	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2555	{
2556	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2557	Let us assert it as such and use native-width VMWRITE. */
2558	X86RFLAGS uRFlags = pCtx->rflags;
2559	Assert(uRFlags.u64 >> 32 == 0);
2560	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2561	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2562
2563	/*
2564	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2565	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2566	*/
2567	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2568	{
2569	Assert(pVM->hm.s.vmx.pRealModeTSS);
2570	Assert(PDMVmmDevHeapIsEnabled(pVM));
2571	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2572	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2573	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2574	}
2575
2576	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2577	AssertRCReturn(rc, rc);
2578
2579	Log(("VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2580	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2581	}
2582	return rc;
2583	}
2584
2585
2586	/**
2587	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2588	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2589	* assembly code.
2590	*
2591	* @returns VBox status code.
2592	* @param pVM Pointer to the VM.
2593	* @param pVCpu Pointer to the VMCPU.
2594	* @param pCtx Pointer to the guest-CPU context.
2595	*
2596	* @remarks No-long-jump zone!!!
2597	*/
2598	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2599	{
2600	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2601	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2602	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2603	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2604	return rc;
2605	}
2606
2607
2608	/**
2609	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2610	* in the VMCS.
2611	*
2612	* @returns VBox status code.
2613	* @param pVM Pointer to the VM.
2614	* @param pVCpu Pointer to the VMCPU.
2615	* @param pCtx Pointer to the guest-CPU context.
2616	*
2617	* @remarks No-long-jump zone!!!
2618	*/
2619	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2620	{
2621	int rc = VINF_SUCCESS;
2622
2623	/*
2624	* Guest CR0.
2625	* Guest FPU.
2626	*/
2627	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2628	{
2629	uint64_t u64GuestCR0 = pCtx->cr0;
2630
2631	/* The guest's view (read access) of its CR0 is unblemished. */
2632	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2633	AssertRCReturn(rc, rc);
2634	Log2(("VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2635
2636	/* Setup VT-x's view of the guest CR0. */
2637	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2638	if (pVM->hm.s.fNestedPaging)
2639	{
2640	if (CPUMIsGuestPagingEnabledEx(pCtx))
2641	{
2642	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2643	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2644	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2645	}
2646	else
2647	{
2648	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2649	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2650	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2651	}
2652
2653	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2654	AssertRCReturn(rc, rc);
2655	}
2656	else
2657	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2658
2659	/*
2660	* Guest FPU bits.
2661	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2662	* CPUs to support VT-x (prob. means all the way up to Nehalem) and no mention of with regards to UX in VM-entry checks.
2663	*/
2664	u64GuestCR0 \|= X86_CR0_NE;
2665	bool fInterceptNM = false;
2666	if (CPUMIsGuestFPUStateActive(pVCpu))
2667	{
2668	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2669	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2670	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2671	}
2672	else
2673	{
2674	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2675	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2676	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2677	}
2678
2679	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2680	bool fInterceptMF = false;
2681	if (!(pCtx->cr0 & X86_CR0_NE))
2682	fInterceptMF = true;
2683
2684	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2685	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2686	{
2687	Assert(PDMVmmDevHeapIsEnabled(pVM));
2688	Assert(pVM->hm.s.vmx.pRealModeTSS);
2689	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2690	fInterceptNM = true;
2691	fInterceptMF = true;
2692	}
2693	else
2694	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2695
2696	if (fInterceptNM)
2697	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2698	else
2699	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2700
2701	if (fInterceptMF)
2702	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2703	else
2704	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2705
2706	/* Additional intercepts for debugging, define these yourself explicitly. */
2707	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2708	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2709	\| RT_BIT(X86_XCPT_DB)
2710	\| RT_BIT(X86_XCPT_DE)
2711	\| RT_BIT(X86_XCPT_NM)
2712	\| RT_BIT(X86_XCPT_UD)
2713	\| RT_BIT(X86_XCPT_NP)
2714	\| RT_BIT(X86_XCPT_SS)
2715	\| RT_BIT(X86_XCPT_GP)
2716	\| RT_BIT(X86_XCPT_PF)
2717	\| RT_BIT(X86_XCPT_MF);
2718	#elif defined(VBOX_ALWAYS_TRAP_PF)
2719	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2720	#endif
2721
2722	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2723
2724	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2725	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2726	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2727	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2728	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2729	else
2730	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2731
2732	u64GuestCR0 \|= uSetCR0;
2733	u64GuestCR0 &= uZapCR0;
2734	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2735
2736	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2737	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2738	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2739	Log2(("VMX_VMCS_GUEST_CR0=%#RX32\n", (uint32_t)u64GuestCR0));
2740
2741	/*
2742	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2743	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2744	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2745	*/
2746	uint64_t u64CR0Mask = 0;
2747	u64CR0Mask = X86_CR0_PE
2748	\| X86_CR0_WP
2749	\| X86_CR0_PG
2750	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2751	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2752	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2753
2754	/* We don't need to intercept changes to CR0.PE with unrestricted guests. */
2755	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2756	u64CR0Mask &= ~X86_CR0_PE;
2757
2758	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2759	if (fInterceptNM)
2760	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2761	else
2762	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2763
2764	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2765	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2766	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2767	AssertRCReturn(rc, rc);
2768
2769	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2770	}
2771
2772	/*
2773	* Guest CR2.
2774	* It's always loaded in the assembler code. Nothing to do here.
2775	*/
2776
2777	/*
2778	* Guest CR3.
2779	*/
2780	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2781	{
2782	uint64_t u64GuestCR3 = 0;
2783	if (pVM->hm.s.fNestedPaging)
2784	{
2785	pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2786
2787	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2788	Assert(pVCpu->hm.s.vmx.GCPhysEPTP);
2789	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff0000000000000ULL));
2790	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2791
2792	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2793	pVCpu->hm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2794	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2795
2796	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2797	AssertMsg( ((pVCpu->hm.s.vmx.GCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2798	&& ((pVCpu->hm.s.vmx.GCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2799	("EPTP %#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2800
2801	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2802	AssertRCReturn(rc, rc);
2803	Log(("VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2804
2805	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2806	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2807	{
2808	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2809	if (CPUMIsGuestInPAEModeEx(pCtx))
2810	{
2811	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2812	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2813	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2814	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2815	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2816	AssertRCReturn(rc, rc);
2817	}
2818
2819	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2820	have Unrestricted Execution to handle the guest when it's not using paging. */
2821	u64GuestCR3 = pCtx->cr3;
2822	}
2823	else
2824	{
2825	/*
2826	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2827	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2828	* EPT takes care of translating it to host-physical addresses.
2829	*/
2830	RTGCPHYS GCPhys;
2831	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2832	Assert(PDMVmmDevHeapIsEnabled(pVM));
2833
2834	/* We obtain it here every time as the guest could have relocated this PCI region. */
2835	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2836	AssertRCReturn(rc, rc);
2837
2838	u64GuestCR3 = GCPhys;
2839	}
2840	}
2841	else
2842	{
2843	/* Non-nested paging case, just use the hypervisor's CR3. */
2844	u64GuestCR3 = PGMGetHyperCR3(pVCpu);
2845	}
2846
2847	Log2(("VMX_VMCS_GUEST_CR3=%#RX64\n", u64GuestCR3));
2848	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, u64GuestCR3);
2849	AssertRCReturn(rc, rc);
2850
2851	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2852	}
2853
2854	/*
2855	* Guest CR4.
2856	*/
2857	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2858	{
2859	uint64_t u64GuestCR4 = pCtx->cr4;
2860
2861	/* The guest's view of its CR4 is unblemished. */
2862	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2863	AssertRCReturn(rc, rc);
2864	Log2(("VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2865
2866	/* Setup VT-x's view of the guest CR4. */
2867	/*
2868	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2869	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2870	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2871	*/
2872	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2873	{
2874	Assert(pVM->hm.s.vmx.pRealModeTSS);
2875	Assert(PDMVmmDevHeapIsEnabled(pVM));
2876	u64GuestCR4 &= ~X86_CR4_VME;
2877	}
2878
2879	if (pVM->hm.s.fNestedPaging)
2880	{
2881	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2882	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2883	{
2884	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2885	u64GuestCR4 \|= X86_CR4_PSE;
2886	/* Our identity mapping is a 32 bits page directory. */
2887	u64GuestCR4 &= ~X86_CR4_PAE;
2888	}
2889	/* else use guest CR4.*/
2890	}
2891	else
2892	{
2893	/*
2894	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2895	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2896	*/
2897	switch (pVCpu->hm.s.enmShadowMode)
2898	{
2899	case PGMMODE_REAL: /* Real-mode. */
2900	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2901	case PGMMODE_32_BIT: /* 32-bit paging. */
2902	{
2903	u64GuestCR4 &= ~X86_CR4_PAE;
2904	break;
2905	}
2906
2907	case PGMMODE_PAE: /* PAE paging. */
2908	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2909	{
2910	u64GuestCR4 \|= X86_CR4_PAE;
2911	break;
2912	}
2913
2914	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2915	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2916	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2917	break;
2918	#endif
2919	default:
2920	AssertFailed();
2921	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2922	}
2923	}
2924
2925	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2926	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2927	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2928	u64GuestCR4 \|= uSetCR4;
2929	u64GuestCR4 &= uZapCR4;
2930
2931	/* Write VT-x's view of the guest CR4 into the VMCS. */
2932	Log2(("VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2933	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2934
2935	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2936	uint64_t u64CR4Mask = 0;
2937	u64CR4Mask = X86_CR4_VME
2938	\| X86_CR4_PAE
2939	\| X86_CR4_PGE
2940	\| X86_CR4_PSE
2941	\| X86_CR4_VMXE;
2942	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
2943	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
2944	AssertRCReturn(rc, rc);
2945
2946	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
2947	}
2948	return rc;
2949	}
2950
2951
2952	/**
2953	* Loads the guest debug registers into the guest-state area in the VMCS.
2954	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
2955	*
2956	* @returns VBox status code.
2957	* @param pVM Pointer to the VM.
2958	* @param pVCpu Pointer to the VMCPU.
2959	* @param pCtx Pointer to the guest-CPU context.
2960	*
2961	* @remarks No-long-jump zone!!!
2962	*/
2963	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2964	{
2965	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
2966	return VINF_SUCCESS;
2967
2968	#ifdef DEBUG
2969	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
2970	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
2971	{
2972	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
2973	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
2974	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
2975	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
2976	}
2977	#endif
2978
2979	int rc = VERR_INTERNAL_ERROR_5;
2980	bool fInterceptDB = false;
2981	bool fInterceptMovDRx = false;
2982	if (DBGFIsStepping(pVCpu))
2983	{
2984	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
2985	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2986	{
2987	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2988	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2989	AssertRCReturn(rc, rc);
2990	Assert(fInterceptDB == false);
2991	}
2992	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
2993	{
2994	Assert(fInterceptDB == false);
2995	/** @todo can we somehow signal DBGF that it cannot single-step instead of
2996	* just continuing? */
2997	}
2998	else
2999	fInterceptDB = true;
3000	}
3001	else
3002	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
3003
3004	/*
3005	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
3006	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
3007	* The same for the hypervisor DRx registers, priority is for the guest here.
3008	*/
3009	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3010	&& !CPUMIsGuestDebugStateActive(pVCpu))
3011	{
3012	/* Save the host and load the guest debug registers. This will make the guest debug state active. */
3013	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3014	AssertRC(rc);
3015	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3016	Assert(fInterceptMovDRx == false);
3017	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3018	}
3019	else if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)
3020	&& !CPUMIsHyperDebugStateActive(pVCpu))
3021	{
3022	/* Save the host and load the hypervisor debug registers. This will make the hyper debug state active. */
3023	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3024	AssertRC(rc);
3025	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3026	fInterceptMovDRx = true;
3027	}
3028	else
3029	Assert(fInterceptMovDRx == false); /* No need to intercept MOV DRx if DBGF is not active nor the guest is debugging. */
3030
3031	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3032	if (fInterceptDB)
3033	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3034	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3035	{
3036	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3037	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3038	#endif
3039	}
3040
3041	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3042	if (fInterceptMovDRx)
3043	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3044	else
3045	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3046
3047	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3048	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3049
3050	/* The guest's view of its DR7 is unblemished. */
3051	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3052
3053	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3054	return rc;
3055	}
3056
3057
3058	#ifdef DEBUG
3059	/**
3060	* Debug function to validate segment registers.
3061	*/
3062	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3063	{
3064	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3065	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3066	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3067	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3068	{
3069	/* Protected mode checks */
3070	/* CS */
3071	Assert(pCtx->cs.Attr.n.u1Present);
3072	Assert(!(pCtx->cs.Attr.u & 0xf00));
3073	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3074	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3075	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3076	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3077	\|\| (pCtx->cs.Attr.n.u1Granularity));
3078	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3079	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3080	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3081	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3082	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3083	else
3084	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3085	/* SS */
3086	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3087	{
3088	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3089	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3090	Assert(pCtx->ss.Attr.n.u1Present);
3091	Assert(!(pCtx->ss.Attr.u & 0xf00));
3092	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3093	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3094	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3095	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3096	\|\| (pCtx->ss.Attr.n.u1Granularity));
3097	}
3098	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3099	/* CR0 might not be up-to-date here always, hence disabled. */
3100	#if 0
3101	if (!pCtx->cr0 & X86_CR0_PE)
3102	Assert(!pCtx->ss.Attr.n.u2Dpl);
3103	#endif
3104	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3105	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3106	{
3107	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3108	Assert(pCtx->ds.Attr.n.u1Present);
3109	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3110	Assert(!(pCtx->ds.Attr.u & 0xf00));
3111	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3112	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3113	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3114	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3115	\|\| (pCtx->ds.Attr.n.u1Granularity));
3116	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3117	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3118	}
3119	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3120	{
3121	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3122	Assert(pCtx->es.Attr.n.u1Present);
3123	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3124	Assert(!(pCtx->es.Attr.u & 0xf00));
3125	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3126	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3127	\|\| !(pCtx->es.Attr.n.u1Granularity));
3128	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3129	\|\| (pCtx->es.Attr.n.u1Granularity));
3130	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3131	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3132	}
3133	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3134	{
3135	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3136	Assert(pCtx->fs.Attr.n.u1Present);
3137	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3138	Assert(!(pCtx->fs.Attr.u & 0xf00));
3139	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3140	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3141	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3142	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3143	\|\| (pCtx->fs.Attr.n.u1Granularity));
3144	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3145	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3146	}
3147	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3148	{
3149	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3150	Assert(pCtx->gs.Attr.n.u1Present);
3151	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3152	Assert(!(pCtx->gs.Attr.u & 0xf00));
3153	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3154	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3155	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3156	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3157	\|\| (pCtx->gs.Attr.n.u1Granularity));
3158	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3159	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3160	}
3161	/* 64-bit capable CPUs. */
3162	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3163	Assert(!(pCtx->cs.u64Base >> 32));
3164	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3165	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3166	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3167	# endif
3168	}
3169	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3170	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3171	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3172	{
3173	/* Real and v86 mode checks. */
3174	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3175	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3176	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3177	{
3178	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3179	}
3180	else
3181	{
3182	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3183	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3184	}
3185
3186	/* CS */
3187	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3188	Assert(pCtx->cs.u32Limit == 0xffff);
3189	Assert(u32CSAttr == 0xf3);
3190	/* SS */
3191	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3192	Assert(pCtx->ss.u32Limit == 0xffff);
3193	Assert(u32SSAttr == 0xf3);
3194	/* DS */
3195	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3196	Assert(pCtx->ds.u32Limit == 0xffff);
3197	Assert(u32DSAttr == 0xf3);
3198	/* ES */
3199	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3200	Assert(pCtx->es.u32Limit == 0xffff);
3201	Assert(u32ESAttr == 0xf3);
3202	/* FS */
3203	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3204	Assert(pCtx->fs.u32Limit == 0xffff);
3205	Assert(u32FSAttr == 0xf3);
3206	/* GS */
3207	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3208	Assert(pCtx->gs.u32Limit == 0xffff);
3209	Assert(u32GSAttr == 0xf3);
3210	/* 64-bit capable CPUs. */
3211	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3212	Assert(!(pCtx->cs.u64Base >> 32));
3213	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3214	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3215	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3216	# endif
3217	}
3218	}
3219	#endif /* DEBUG */
3220
3221
3222	/**
3223	* Writes a guest segment register into the guest-state area in the VMCS.
3224	*
3225	* @returns VBox status code.
3226	* @param pVM Pointer to the VM.
3227	* @param pVCpu Pointer to the VMCPU.
3228	* @param idxSel Index of the selector in the VMCS.
3229	* @param idxLimit Index of the segment limit in the VMCS.
3230	* @param idxBase Index of the segment base in the VMCS.
3231	* @param idxAccess Index of the access rights of the segment in the VMCS.
3232	* @param pSelReg Pointer to the segment selector.
3233	* @param pCtx Pointer to the guest-CPU context.
3234	*
3235	* @remarks No-long-jump zone!!!
3236	*/
3237	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3238	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3239	{
3240	int rc;
3241	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3242	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3243	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3244	AssertRCReturn(rc, rc);
3245
3246	uint32_t u32Access = pSelReg->Attr.u;
3247	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3248	{
3249	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3250	u32Access = 0xf3;
3251	Assert(pVM->hm.s.vmx.pRealModeTSS);
3252	Assert(PDMVmmDevHeapIsEnabled(pVM));
3253	}
3254	else
3255	{
3256	/*
3257	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3258	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3259	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3260	* loaded in protected-mode have their attribute as 0.
3261	*/
3262	if (!u32Access)
3263	u32Access = VMX_SEL_UNUSABLE;
3264	}
3265
3266	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3267	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3268	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3269
3270	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3271	AssertRCReturn(rc, rc);
3272	return rc;
3273	}
3274
3275
3276	/**
3277	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3278	* into the guest-state area in the VMCS.
3279	*
3280	* @returns VBox status code.
3281	* @param pVM Pointer to the VM.
3282	* @param pVCPU Pointer to the VMCPU.
3283	* @param pCtx Pointer to the guest-CPU context.
3284	*
3285	* @remarks No-long-jump zone!!!
3286	* @remarks Requires RFLAGS (for debug assertions).
3287	*/
3288	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3289	{
3290	int rc = VERR_INTERNAL_ERROR_5;
3291
3292	/*
3293	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3294	*/
3295	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3296	{
3297	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3298	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3299	{
3300	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3301	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3302	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3303	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3304	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3305	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3306	}
3307
3308	#ifdef VBOX_WITH_REM
3309	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3310	{
3311	Assert(pVM->hm.s.vmx.pRealModeTSS);
3312	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3313	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3314	{
3315	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3316	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3317	&& enmGuestMode >= PGMMODE_PROTECTED)
3318	{
3319	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3320	in real-mode (e.g. OpenBSD 4.0) */
3321	REMFlushTBs(pVM);
3322	Log2(("Switch to protected mode detected!\n"));
3323	}
3324	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3325	}
3326	}
3327	#endif
3328	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3329	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3330	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3331	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3332	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3333	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3334	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3335	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3336	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3337	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3338	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3339	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3340	AssertRCReturn(rc, rc);
3341
3342	#ifdef DEBUG
3343	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3344	#endif
3345	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3346	}
3347
3348	/*
3349	* Guest TR.
3350	*/
3351	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3352	{
3353	/*
3354	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3355	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3356	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3357	*/
3358	uint16_t u16Sel = 0;
3359	uint32_t u32Limit = 0;
3360	uint64_t u64Base = 0;
3361	uint32_t u32AccessRights = 0;
3362
3363	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3364	{
3365	u16Sel = pCtx->tr.Sel;
3366	u32Limit = pCtx->tr.u32Limit;
3367	u64Base = pCtx->tr.u64Base;
3368	u32AccessRights = pCtx->tr.Attr.u;
3369	}
3370	else
3371	{
3372	Assert(pVM->hm.s.vmx.pRealModeTSS);
3373	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3374
3375	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3376	RTGCPHYS GCPhys;
3377	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3378	AssertRCReturn(rc, rc);
3379
3380	X86DESCATTR DescAttr;
3381	DescAttr.u = 0;
3382	DescAttr.n.u1Present = 1;
3383	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3384
3385	u16Sel = 0;
3386	u32Limit = HM_VTX_TSS_SIZE;
3387	u64Base = GCPhys; /* in real-mode phys = virt. */
3388	u32AccessRights = DescAttr.u;
3389	}
3390
3391	/* Validate. */
3392	Assert(!(u16Sel & RT_BIT(2)));
3393	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3394	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3395	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3396	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3397	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3398	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3399	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3400	Assert( (u32Limit & 0xfff) == 0xfff
3401	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3402	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3403	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3404
3405	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3406	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3407	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3408	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3409	AssertRCReturn(rc, rc);
3410
3411	Log2(("VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3412	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3413	}
3414
3415	/*
3416	* Guest GDTR.
3417	*/
3418	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3419	{
3420	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3421	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3422	AssertRCReturn(rc, rc);
3423
3424	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3425	Log2(("VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3426	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3427	}
3428
3429	/*
3430	* Guest LDTR.
3431	*/
3432	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3433	{
3434	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3435	uint32_t u32Access = 0;
3436	if (!(pCtx->ldtr.Attr.u & VMX_SEL_UNUSABLE))
3437	u32Access = VMX_SEL_UNUSABLE;
3438	else
3439	u32Access = pCtx->ldtr.Attr.u;
3440
3441	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3442	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3443	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3444	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3445	AssertRCReturn(rc, rc);
3446
3447	/* Validate. */
3448	if (!(u32Access & VMX_SEL_UNUSABLE))
3449	{
3450	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3451	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3452	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3453	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3454	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3455	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3456	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3457	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3458	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3459	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3460	}
3461
3462	Log2(("VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3463	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3464	}
3465
3466	/*
3467	* Guest IDTR.
3468	*/
3469	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3470	{
3471	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3472	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3473	AssertRCReturn(rc, rc);
3474
3475	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3476	Log2(("VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3477	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3478	}
3479
3480	/*
3481	* Guest FS & GS base MSRs.
3482	* We already initialized the FS & GS base as part of the guest segment registers, but the guest's FS/GS base
3483	* MSRs might have changed (e.g. due to WRMSR) and we need to update the bases if that happened. These MSRs
3484	* are only available in 64-bit mode.
3485	*/
3486	/** @todo Avoid duplication of this code in assembly (see MYPUSHSEGS) - it
3487	* should not be necessary to do it in assembly again. */
3488	if (CPUMIsGuestInLongModeEx(pCtx))
3489	{
3490	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FS_BASE_MSR)
3491	{
3492	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
3493	AssertRCReturn(rc, rc);
3494	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_FS_BASE_MSR;
3495	}
3496
3497	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GS_BASE_MSR)
3498	{
3499	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
3500	AssertRCReturn(rc, rc);
3501	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GS_BASE_MSR;
3502	}
3503	}
3504	else
3505	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_FS_BASE_MSR \| HM_CHANGED_GUEST_GS_BASE_MSR);
3506
3507	return VINF_SUCCESS;
3508	}
3509
3510
3511	/**
3512	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3513	* areas. These MSRs will automatically be loaded to the host CPU on every
3514	* successful VM entry and stored from the host CPU on every successful VM exit.
3515	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3516	*
3517	* @returns VBox status code.
3518	* @param pVM Pointer to the VM.
3519	* @param pVCpu Pointer to the VMCPU.
3520	* @param pCtx Pointer to the guest-CPU context.
3521	*
3522	* @remarks No-long-jump zone!!!
3523	*/
3524	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3525	{
3526	AssertPtr(pVCpu);
3527	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3528
3529	/*
3530	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3531	*/
3532	int rc = VINF_SUCCESS;
3533	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3534	{
3535	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3536	unsigned cGuestMsrs = 0;
3537
3538	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3539	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3540	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3541	if (fSupportsNX \|\| fSupportsLongMode)
3542	{
3543	/** @todo support save IA32_EFER, i.e.
3544	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3545	* guest EFER need not be part of the VM-entry MSR-load area. */
3546	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3547	pGuestMsr->u32Reserved = 0;
3548	pGuestMsr->u64Value = pCtx->msrEFER;
3549	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3550	if (!CPUMIsGuestInLongModeEx(pCtx))
3551	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3552	pGuestMsr++; cGuestMsrs++;
3553	if (fSupportsLongMode)
3554	{
3555	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3556	pGuestMsr->u32Reserved = 0;
3557	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3558	pGuestMsr++; cGuestMsrs++;
3559	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3560	pGuestMsr->u32Reserved = 0;
3561	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3562	pGuestMsr++; cGuestMsrs++;
3563	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3564	pGuestMsr->u32Reserved = 0;
3565	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3566	pGuestMsr++; cGuestMsrs++;
3567	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3568	#if 0
3569	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3570	pGuestMsr->u32Reserved = 0;
3571	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3572	pGuestMsr++; cGuestMsrs++;
3573	#endif
3574	}
3575	}
3576
3577	/*
3578	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3579	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3580	*/
3581	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3582	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3583	{
3584	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3585	pGuestMsr->u32Reserved = 0;
3586	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3587	AssertRCReturn(rc, rc);
3588	pGuestMsr++; cGuestMsrs++;
3589	}
3590
3591	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3592	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3593	{
3594	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3595	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3596	}
3597
3598	/* Update the VCPU's copy of the guest MSR count. */
3599	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3600	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3601	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3602	AssertRCReturn(rc, rc);
3603
3604	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3605	}
3606
3607	/*
3608	* Guest Sysenter MSRs.
3609	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3610	* VM exits on WRMSRs for these MSRs.
3611	*/
3612	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3613	{
3614	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3615	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3616	}
3617	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3618	{
3619	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3620	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3621	}
3622	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3623	{
3624	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3625	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3626	}
3627	AssertRCReturn(rc, rc);
3628
3629	return rc;
3630	}
3631
3632
3633	/**
3634	* Loads the guest activity state into the guest-state area in the VMCS.
3635	*
3636	* @returns VBox status code.
3637	* @param pVM Pointer to the VM.
3638	* @param pVCpu Pointer to the VMCPU.
3639	* @param pCtx Pointer to the guest-CPU context.
3640	*
3641	* @remarks No-long-jump zone!!!
3642	*/
3643	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3644	{
3645	/** @todo See if we can make use of other states, e.g.
3646	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3647	int rc = VINF_SUCCESS;
3648	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3649	{
3650	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3651	AssertRCReturn(rc, rc);
3652	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3653	}
3654	return rc;
3655	}
3656
3657
3658	/**
3659	* Sets up the appropriate function to run guest code.
3660	*
3661	* @returns VBox status code.
3662	* @param pVM Pointer to the VM.
3663	* @param pVCpu Pointer to the VMCPU.
3664	* @param pCtx Pointer to the guest-CPU context.
3665	*
3666	* @remarks No-long-jump zone!!!
3667	*/
3668	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3669	{
3670	if (CPUMIsGuestInLongModeEx(pCtx))
3671	{
3672	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3673	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3674	#endif
3675	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3676	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3677	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3678	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3679	#else
3680	/* 64-bit host or hybrid host. */
3681	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3682	#endif
3683	}
3684	else
3685	{
3686	/* Guest is not in long mode, use the 32-bit handler. */
3687	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3688	}
3689	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3690	return VINF_SUCCESS;
3691	}
3692
3693
3694	/**
3695	* Wrapper for running the guest code in VT-x.
3696	*
3697	* @returns VBox strict status code.
3698	* @param pVM Pointer to the VM.
3699	* @param pVCpu Pointer to the VMCPU.
3700	* @param pCtx Pointer to the guest-CPU context.
3701	*
3702	* @remarks No-long-jump zone!!!
3703	*/
3704	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3705	{
3706	/*
3707	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3708	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3709	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3710	*/
3711	#ifdef VBOX_WITH_KERNEL_USING_XMM
3712	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3713	#else
3714	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3715	#endif
3716	}
3717
3718
3719	/**
3720	* Report world-switch error and dump some useful debug info.
3721	*
3722	* @param pVM Pointer to the VM.
3723	* @param pVCpu Pointer to the VMCPU.
3724	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3725	* @param pCtx Pointer to the guest-CPU context.
3726	* @param pVmxTransient Pointer to the VMX transient structure (only
3727	* exitReason updated).
3728	*/
3729	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3730	{
3731	Assert(pVM);
3732	Assert(pVCpu);
3733	Assert(pCtx);
3734	Assert(pVmxTransient);
3735	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3736
3737	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3738	switch (rcVMRun)
3739	{
3740	case VERR_VMX_INVALID_VMXON_PTR:
3741	AssertFailed();
3742	break;
3743	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3744	case VERR_VMX_UNABLE_TO_START_VM:
3745	case VERR_VMX_UNABLE_TO_RESUME_VM:
3746	{
3747	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3748	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3749	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3750	AssertRC(rc);
3751
3752	#ifdef VBOX_STRICT
3753	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3754	pVmxTransient->uExitReason));
3755	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3756	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3757	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3758	Log(("InstrError Desc. \"%s\"\n", s_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3759	else
3760	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3761
3762	/* VMX control bits. */
3763	uint32_t u32Val;
3764	uint64_t u64Val;
3765	RTHCUINTREG uHCReg;
3766	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3767	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3768	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3769	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3770	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3771	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3772	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3773	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3774	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3775	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3776	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3777	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3778	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3779	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3780	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3781	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3782	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3783	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3784	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3785	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3786	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3787	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3788	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3789	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3790	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3791	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3792	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3793	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3794	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3795	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3796	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3797	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3798	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3799	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3800	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3801	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3802	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3803	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3804	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3805	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3806	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3807	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3808
3809	/* Guest bits. */
3810	RTGCUINTREG uGCReg;
3811	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3812	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3813	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3814	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3815	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3816	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3817	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3818	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3819
3820	/* Host bits. */
3821	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3822	Log(("Host CR0 %#RHr\n", uHCReg));
3823	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3824	Log(("Host CR3 %#RHr\n", uHCReg));
3825	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3826	Log(("Host CR4 %#RHr\n", uHCReg));
3827
3828	RTGDTR HostGdtr;
3829	PCX86DESCHC pDesc;
3830	ASMGetGDTR(&HostGdtr);
3831	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3832	Log(("Host CS %#08x\n", u32Val));
3833	if (u32Val < HostGdtr.cbGdt)
3834	{
3835	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3836	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3837	}
3838
3839	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3840	Log(("Host DS %#08x\n", u32Val));
3841	if (u32Val < HostGdtr.cbGdt)
3842	{
3843	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3844	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3845	}
3846
3847	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3848	Log(("Host ES %#08x\n", u32Val));
3849	if (u32Val < HostGdtr.cbGdt)
3850	{
3851	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3852	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3853	}
3854
3855	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3856	Log(("Host FS %#08x\n", u32Val));
3857	if (u32Val < HostGdtr.cbGdt)
3858	{
3859	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3860	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3861	}
3862
3863	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3864	Log(("Host GS %#08x\n", u32Val));
3865	if (u32Val < HostGdtr.cbGdt)
3866	{
3867	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3868	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3869	}
3870
3871	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3872	Log(("Host SS %#08x\n", u32Val));
3873	if (u32Val < HostGdtr.cbGdt)
3874	{
3875	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3876	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3877	}
3878
3879	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3880	Log(("Host TR %#08x\n", u32Val));
3881	if (u32Val < HostGdtr.cbGdt)
3882	{
3883	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3884	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3885	}
3886
3887	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3888	Log(("Host TR Base %#RHv\n", uHCReg));
3889	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3890	Log(("Host GDTR Base %#RHv\n", uHCReg));
3891	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3892	Log(("Host IDTR Base %#RHv\n", uHCReg));
3893	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3894	Log(("Host SYSENTER CS %#08x\n", u32Val));
3895	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3896	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3897	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3898	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3899	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3900	Log(("Host RSP %#RHv\n", uHCReg));
3901	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3902	Log(("Host RIP %#RHv\n", uHCReg));
3903	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3904	if (VMX_IS_64BIT_HOST_MODE())
3905	{
3906	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3907	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3908	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3909	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3910	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3911	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3912	}
3913	# endif
3914	#endif /* VBOX_STRICT */
3915	break;
3916	}
3917
3918	default:
3919	/* Impossible */
3920	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3921	break;
3922	}
3923	NOREF(pVM);
3924	}
3925
3926
3927	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3928	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3929	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3930	#endif
3931
3932	#ifdef VBOX_STRICT
3933	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3934	{
3935	switch (idxField)
3936	{
3937	case VMX_VMCS_GUEST_RIP:
3938	case VMX_VMCS_GUEST_RSP:
3939	case VMX_VMCS_GUEST_RFLAGS:
3940	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
3941	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
3942	case VMX_VMCS_GUEST_CR0:
3943	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
3944	case VMX_VMCS_GUEST_CR4:
3945	case VMX_VMCS_GUEST_DR7:
3946	case VMX_VMCS32_GUEST_SYSENTER_CS:
3947	case VMX_VMCS_GUEST_SYSENTER_EIP:
3948	case VMX_VMCS_GUEST_SYSENTER_ESP:
3949	case VMX_VMCS32_GUEST_GDTR_LIMIT:
3950	case VMX_VMCS_GUEST_GDTR_BASE:
3951	case VMX_VMCS32_GUEST_IDTR_LIMIT:
3952	case VMX_VMCS_GUEST_IDTR_BASE:
3953	case VMX_VMCS16_GUEST_FIELD_CS:
3954	case VMX_VMCS32_GUEST_CS_LIMIT:
3955	case VMX_VMCS_GUEST_CS_BASE:
3956	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
3957	case VMX_VMCS16_GUEST_FIELD_DS:
3958	case VMX_VMCS32_GUEST_DS_LIMIT:
3959	case VMX_VMCS_GUEST_DS_BASE:
3960	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
3961	case VMX_VMCS16_GUEST_FIELD_ES:
3962	case VMX_VMCS32_GUEST_ES_LIMIT:
3963	case VMX_VMCS_GUEST_ES_BASE:
3964	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
3965	case VMX_VMCS16_GUEST_FIELD_FS:
3966	case VMX_VMCS32_GUEST_FS_LIMIT:
3967	case VMX_VMCS_GUEST_FS_BASE:
3968	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
3969	case VMX_VMCS16_GUEST_FIELD_GS:
3970	case VMX_VMCS32_GUEST_GS_LIMIT:
3971	case VMX_VMCS_GUEST_GS_BASE:
3972	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
3973	case VMX_VMCS16_GUEST_FIELD_SS:
3974	case VMX_VMCS32_GUEST_SS_LIMIT:
3975	case VMX_VMCS_GUEST_SS_BASE:
3976	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
3977	case VMX_VMCS16_GUEST_FIELD_LDTR:
3978	case VMX_VMCS32_GUEST_LDTR_LIMIT:
3979	case VMX_VMCS_GUEST_LDTR_BASE:
3980	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
3981	case VMX_VMCS16_GUEST_FIELD_TR:
3982	case VMX_VMCS32_GUEST_TR_LIMIT:
3983	case VMX_VMCS_GUEST_TR_BASE:
3984	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
3985	case VMX_VMCS32_RO_EXIT_REASON:
3986	case VMX_VMCS32_RO_VM_INSTR_ERROR:
3987	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
3988	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
3989	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
3990	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
3991	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3992	case VMX_VMCS32_RO_IDT_INFO:
3993	case VMX_VMCS32_RO_IDT_ERROR_CODE:
3994	case VMX_VMCS_GUEST_CR3:
3995	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
3996	return true;
3997	}
3998	return false;
3999	}
4000
4001	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4002	{
4003	switch (idxField)
4004	{
4005	case VMX_VMCS_GUEST_LDTR_BASE:
4006	case VMX_VMCS_GUEST_TR_BASE:
4007	case VMX_VMCS_GUEST_GDTR_BASE:
4008	case VMX_VMCS_GUEST_IDTR_BASE:
4009	case VMX_VMCS_GUEST_SYSENTER_EIP:
4010	case VMX_VMCS_GUEST_SYSENTER_ESP:
4011	case VMX_VMCS_GUEST_CR0:
4012	case VMX_VMCS_GUEST_CR4:
4013	case VMX_VMCS_GUEST_CR3:
4014	case VMX_VMCS_GUEST_DR7:
4015	case VMX_VMCS_GUEST_RIP:
4016	case VMX_VMCS_GUEST_RSP:
4017	case VMX_VMCS_GUEST_CS_BASE:
4018	case VMX_VMCS_GUEST_DS_BASE:
4019	case VMX_VMCS_GUEST_ES_BASE:
4020	case VMX_VMCS_GUEST_FS_BASE:
4021	case VMX_VMCS_GUEST_GS_BASE:
4022	case VMX_VMCS_GUEST_SS_BASE:
4023	return true;
4024	}
4025	return false;
4026	}
4027	#endif /* VBOX_STRICT */
4028
4029	/**
4030	* Executes the specified handler in 64-bit mode.
4031	*
4032	* @returns VBox status code.
4033	* @param pVM Pointer to the VM.
4034	* @param pVCpu Pointer to the VMCPU.
4035	* @param pCtx Pointer to the guest CPU context.
4036	* @param pfnHandler Pointer to the RC handler function.
4037	* @param cbParam Number of parameters.
4038	* @param paParam Array of 32-bit parameters.
4039	*/
4040	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
4041	uint32_t *paParam)
4042	{
4043	int rc, rc2;
4044	PHMGLOBLCPUINFO pCpu;
4045	RTHCPHYS HCPhysCpuPage;
4046	RTHCUINTREG uOldEFlags;
4047
4048	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4049	Assert(pfnHandler);
4050	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4051	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4052
4053	#ifdef VBOX_STRICT
4054	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4055	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4056
4057	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4058	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4059	#endif
4060
4061	/* Disable interrupts. */
4062	uOldEFlags = ASMIntDisableFlags();
4063
4064	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4065	RTCPUID idHostCpu = RTMpCpuId();
4066	CPUMR0SetLApic(pVM, idHostCpu);
4067	#endif
4068
4069	pCpu = HMR0GetCurrentCpu();
4070	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4071
4072	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4073	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4074
4075	/* Leave VMX Root Mode. */
4076	VMXDisable();
4077
4078	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4079
4080	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4081	CPUMSetHyperEIP(pVCpu, pfnHandler);
4082	for (int i = (int)cbParam - 1; i >= 0; i--)
4083	CPUMPushHyper(pVCpu, paParam[i]);
4084
4085	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4086
4087	/* Call the switcher. */
4088	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4089	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4090
4091	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4092	/* Make sure the VMX instructions don't cause #UD faults. */
4093	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4094
4095	/* Re-enter VMX Root Mode */
4096	rc2 = VMXEnable(HCPhysCpuPage);
4097	if (RT_FAILURE(rc2))
4098	{
4099	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4100	ASMSetFlags(uOldEFlags);
4101	return VERR_VMX_VMXON_FAILED;
4102	}
4103
4104	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4105	AssertRC(rc2);
4106	Assert(!(ASMGetFlags() & X86_EFL_IF));
4107	ASMSetFlags(uOldEFlags);
4108	return rc;
4109	}
4110
4111
4112	/**
4113	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4114	* supporting 64-bit guests.
4115	*
4116	* @returns VBox status code.
4117	* @param fResume Whether to VMLAUNCH or VMRESUME.
4118	* @param pCtx Pointer to the guest-CPU context.
4119	* @param pCache Pointer to the VMCS cache.
4120	* @param pVM Pointer to the VM.
4121	* @param pVCpu Pointer to the VMCPU.
4122	*/
4123	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4124	{
4125	uint32_t aParam[6];
4126	PHMGLOBLCPUINFO pCpu = NULL;
4127	RTHCPHYS HCPhysCpuPage = 0;
4128	int rc = VERR_INTERNAL_ERROR_5;
4129
4130	pCpu = HMR0GetCurrentCpu();
4131	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4132
4133	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4134	pCache->uPos = 1;
4135	pCache->interPD = PGMGetInterPaeCR3(pVM);
4136	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4137	#endif
4138
4139	#ifdef DEBUG
4140	pCache->TestIn.HCPhysCpuPage= 0;
4141	pCache->TestIn.HCPhysVmcs = 0;
4142	pCache->TestIn.pCache = 0;
4143	pCache->TestOut.HCPhysVmcs = 0;
4144	pCache->TestOut.pCache = 0;
4145	pCache->TestOut.pCtx = 0;
4146	pCache->TestOut.eflags = 0;
4147	#endif
4148
4149	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4150	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4151	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4152	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4153	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4154	aParam[5] = 0;
4155
4156	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4157	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4158	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4159	#endif
4160	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4161
4162	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4163	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4164	Assert(pCtx->dr[4] == 10);
4165	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4166	#endif
4167
4168	#ifdef DEBUG
4169	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4170	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4171	pVCpu->hm.s.vmx.HCPhysVmcs));
4172	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4173	pCache->TestOut.HCPhysVmcs));
4174	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4175	pCache->TestOut.pCache));
4176	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4177	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4178	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4179	pCache->TestOut.pCtx));
4180	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4181	#endif
4182	return rc;
4183	}
4184
4185
4186	/**
4187	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4188	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4189	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4190	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4191	*
4192	* @returns VBox status code.
4193	* @param pVM Pointer to the VM.
4194	* @param pVCpu Pointer to the VMCPU.
4195	*/
4196	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4197	{
4198	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4199	{ \
4200	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4201	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4202	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4203	}
4204
4205	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4206	{ \
4207	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4208	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4209	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4210	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4211	}
4212
4213	AssertPtr(pVM);
4214	AssertPtr(pVCpu);
4215	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4216
4217	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4218	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4219	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4220	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4221	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4222	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4223	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4224	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4225	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4226
4227	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4228	#if 0
4229	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4230	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4231	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4232	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4233	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4234	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4235	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4236	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4237	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4238	#endif
4239
4240	/* 32-bit guest-state fields. */
4241	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4242	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4243	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4244	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4245	/* Unused 32-bit guest-state fields. */
4246	#if 0
4247	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4248	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4249	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE);
4250	#endif
4251
4252	/* Natural width guest-state fields. */
4253	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4254	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4255	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4256	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4257	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4258	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4259	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4260	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4261	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4262	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4263	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4264	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4265	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4267	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4268	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4269	/* Unused natural width guest-state fields. */
4270	#if 0
4271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4273	#endif
4274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4275	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4276
4277	if (pVM->hm.s.fNestedPaging)
4278	{
4279	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4281	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4282	}
4283	else
4284	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4285
4286	#undef VMXLOCAL_INIT_VMCS_SELREG
4287	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4288	return VINF_SUCCESS;
4289	}
4290
4291
4292	/**
4293	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4294	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4295	* darwin, running 64-bit guests).
4296	*
4297	* @returns VBox status code.
4298	* @param pVCpu Pointer to the VMCPU.
4299	* @param idxField The VMCS field encoding.
4300	* @param u64Val 16, 32 or 64 bits value.
4301	*/
4302	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4303	{
4304	int rc;
4305	switch (idxField)
4306	{
4307	/*
4308	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4309	*/
4310	/* 64-bit Control fields. */
4311	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4312	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4313	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4314	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4315	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4316	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4317	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4318	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4319	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4320	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4321	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4322	case VMX_VMCS64_CTRL_EPTP_FULL:
4323	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4324	/* 64-bit Read-only data fields. */
4325	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4326	/* 64-bit Guest-state fields. */
4327	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4328	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4329	case VMX_VMCS64_GUEST_PAT_FULL:
4330	case VMX_VMCS64_GUEST_EFER_FULL:
4331	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4332	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4333	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4334	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4335	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4336	/* 64-bit Host-state fields. */
4337	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4338	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4339	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4340	{
4341	rc = VMXWriteVmcs32(idxField, u64Val);
4342	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4343	break;
4344	}
4345
4346	/*
4347	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4348	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4349	*/
4350	/* Natural-width Guest-state fields. */
4351	case VMX_VMCS_GUEST_CR0:
4352	case VMX_VMCS_GUEST_CR3:
4353	case VMX_VMCS_GUEST_CR4:
4354	case VMX_VMCS_GUEST_ES_BASE:
4355	case VMX_VMCS_GUEST_CS_BASE:
4356	case VMX_VMCS_GUEST_SS_BASE:
4357	case VMX_VMCS_GUEST_DS_BASE:
4358	case VMX_VMCS_GUEST_FS_BASE:
4359	case VMX_VMCS_GUEST_GS_BASE:
4360	case VMX_VMCS_GUEST_LDTR_BASE:
4361	case VMX_VMCS_GUEST_TR_BASE:
4362	case VMX_VMCS_GUEST_GDTR_BASE:
4363	case VMX_VMCS_GUEST_IDTR_BASE:
4364	case VMX_VMCS_GUEST_DR7:
4365	case VMX_VMCS_GUEST_RSP:
4366	case VMX_VMCS_GUEST_RIP:
4367	case VMX_VMCS_GUEST_RFLAGS:
4368	case VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS:
4369	case VMX_VMCS_GUEST_SYSENTER_ESP:
4370	case VMX_VMCS_GUEST_SYSENTER_EIP:
4371	{
4372	if ((u64Val >> 32ULL) == 0)
4373	{
4374	/* If this field is 64-bit, VT-x will zero out the top bits. */
4375	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4376	}
4377	else
4378	{
4379	/* Assert that only the 32->64 switcher case should ever come here. */
4380	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4381	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4382	}
4383	break;
4384	}
4385
4386	default:
4387	{
4388	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4389	rc = VERR_INVALID_PARAMETER;
4390	break;
4391	}
4392	}
4393	AssertRCReturn(rc, rc);
4394	return rc;
4395	}
4396
4397
4398	/**
4399	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4400	* hosts (except darwin) for 64-bit guests.
4401	*
4402	* @param pVCpu Pointer to the VMCPU.
4403	* @param idxField The VMCS field encoding.
4404	* @param u64Val 16, 32 or 64 bits value.
4405	*/
4406	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4407	{
4408	AssertPtr(pVCpu);
4409	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4410
4411	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4412	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4413
4414	/* Make sure there are no duplicates. */
4415	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
4416	{
4417	if (pCache->Write.aField[i] == idxField)
4418	{
4419	pCache->Write.aFieldVal[i] = u64Val;
4420	return VINF_SUCCESS;
4421	}
4422	}
4423
4424	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4425	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4426	pCache->Write.cValidEntries++;
4427	return VINF_SUCCESS;
4428	}
4429
4430
4431	/**
4432	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4433	*
4434	* @param pVCpu Pointer to the VMCPU.
4435	* @param pCache Pointer to the VMCS cache.
4436	*
4437	* @remarks No-long-jump zone!!!
4438	*/
4439	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4440	{
4441	AssertPtr(pCache);
4442	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4443	{
4444	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4445	AssertRC(rc);
4446	}
4447	pCache->Write.cValidEntries = 0;
4448	}
4449
4450
4451	/**
4452	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4453	*
4454	* @param pVCpu Pointer to the VMCPU.
4455	* @param pCache Pointer to the VMCS cache.
4456	*
4457	* @remarks No-long-jump zone!!!
4458	*/
4459	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4460	{
4461	AssertPtr(pCache);
4462	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4463	{
4464	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4465	AssertRC(rc);
4466	}
4467	}
4468	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4469
4470
4471	/**
4472	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4473	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4474	* timer.
4475	*
4476	* @returns VBox status code.
4477	* @param pVM Pointer to the VM.
4478	* @param pVCpu Pointer to the VMCPU.
4479	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4480	* out-of-sync. Make sure to update the required fields
4481	* before using them.
4482	* @remarks No-long-jump zone!!!
4483	*/
4484	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4485	{
4486	int rc = VERR_INTERNAL_ERROR_5;
4487	bool fOffsettedTsc = false;
4488	if (pVM->hm.s.vmx.fUsePreemptTimer)
4489	{
4490	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4491
4492	/* Make sure the returned values have sane upper and lower boundaries. */
4493	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4494	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4495	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4496	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4497
4498	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4499	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4500	}
4501	else
4502	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4503
4504	if (fOffsettedTsc)
4505	{
4506	uint64_t u64CurTSC = ASMReadTSC();
4507	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4508	{
4509	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4510	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4511
4512	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4513	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4514	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4515	}
4516	else
4517	{
4518	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4519	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4520	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4521	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4522	}
4523	}
4524	else
4525	{
4526	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4527	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4528	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4529	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4530	}
4531	}
4532
4533
4534	/**
4535	* Determines if an exception is a benign exception. Benign exceptions
4536	* are ones which cannot cause double-faults.
4537	*
4538	* @returns true if the exception is benign, false otherwise.
4539	* @param uVector The exception vector.
4540	*/
4541	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint32_t uVector)
4542	{
4543	switch (uVector)
4544	{
4545	case X86_XCPT_DB:
4546	case X86_XCPT_NMI:
4547	case X86_XCPT_BP:
4548	case X86_XCPT_OF:
4549	case X86_XCPT_BR:
4550	case X86_XCPT_UD:
4551	case X86_XCPT_NM:
4552	case X86_XCPT_CO_SEG_OVERRUN:
4553	case X86_XCPT_MF:
4554	case X86_XCPT_AC:
4555	case X86_XCPT_MC:
4556	case X86_XCPT_XF:
4557	return true;
4558	default:
4559	return false;
4560	}
4561	}
4562
4563
4564	/**
4565	* Determines if an exception is a contributory exception. Contributory
4566	* exceptions are ones which can cause double-faults.
4567	*
4568	* @returns true if the exception is contributory, false otherwise.
4569	* @param uVector The exception vector.
4570	*/
4571	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4572	{
4573	switch (uVector)
4574	{
4575	case X86_XCPT_GP:
4576	case X86_XCPT_SS:
4577	case X86_XCPT_NP:
4578	case X86_XCPT_TS:
4579	case X86_XCPT_DE:
4580	return true;
4581	default:
4582	return false;
4583	}
4584	return false;
4585	}
4586
4587
4588	/**
4589	* Determines if we are intercepting any contributory exceptions.
4590	*
4591	* @returns true if we are intercepting any contributory exception, false
4592	* otherwise.
4593	* @param pVCpu Pointer to the VMCPU.
4594	*/
4595	DECLINLINE(bool) hmR0VmxInterceptingContributoryXcpts(PVMCPU pVCpu)
4596	{
4597	if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_GP))
4598	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_SS))
4599	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_NP))
4600	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_TS))
4601	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_DE)))
4602	{
4603	return true;
4604	}
4605	return false;
4606	}
4607
4608
4609	/**
4610	* Handle a condition that occurred while delivering an event through the guest
4611	* IDT.
4612	*
4613	* @returns VBox status code (informational error codes included).
4614	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4615	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4616	* continue execution of the guest which will delivery the #DF.
4617	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4618	*
4619	* @param pVM Pointer to the VM.
4620	* @param pVCpu Pointer to the VMCPU.
4621	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4622	* out-of-sync. Make sure to update the required fields
4623	* before using them.
4624	* @param pVmxTransient Pointer to the VMX transient structure.
4625	*
4626	* @remarks No-long-jump zone!!!
4627	*/
4628	static int hmR0VmxCheckExitDueToEventDelivery(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4629	{
4630	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4631	AssertRCReturn(rc, rc);
4632	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4633	{
4634	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4635	AssertRCReturn(rc, rc);
4636
4637	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4638	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4639	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4640
4641	typedef enum
4642	{
4643	VMXREFLECTXCPT_XCPT, /* Reflect Idt-vectoring exception. */
4644	VMXREFLECTXCPT_DF, /* Reflect a double-fault to the guest. */
4645	VMXREFLECTXCPT_TF, /* Reflect a triple fault state to the VMM. */
4646	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4647	} VMXREFLECTXCPT;
4648
4649	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4650	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4651	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4652	{
4653	if ( hmR0VmxIsBenignXcpt(uIdtVector)
4654	\|\| hmR0VmxIsBenignXcpt(uExitVector)
4655	\|\| ( hmR0VmxIsContributoryXcpt(uIdtVector)
4656	&& uExitVector == X86_XCPT_PF))
4657	{
4658	enmReflect = VMXREFLECTXCPT_XCPT;
4659	}
4660	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF))
4661	&& uIdtVector == X86_XCPT_PF
4662	&& uExitVector == X86_XCPT_PF)
4663	{
4664	pVmxTransient->fVectoringPF = true;
4665	}
4666	else if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4667	&& hmR0VmxIsContributoryXcpt(uExitVector))
4668	{
4669	enmReflect = VMXREFLECTXCPT_DF;
4670	}
4671	else if ( hmR0VmxInterceptingContributoryXcpts(pVCpu)
4672	&& uIdtVector == X86_XCPT_PF
4673	&& hmR0VmxIsContributoryXcpt(uExitVector))
4674	{
4675	enmReflect = VMXREFLECTXCPT_DF;
4676	}
4677	else if (uIdtVector == X86_XCPT_DF)
4678	enmReflect = VMXREFLECTXCPT_TF;
4679	}
4680	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4681	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4682	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4683	{
4684	/*
4685	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4686	* (whatever they are) as they reoccur when restarting the instruction.
4687	*/
4688	enmReflect = VMXREFLECTXCPT_XCPT;
4689	}
4690
4691	Assert(pVmxTransient->fVectoringPF == false \|\| enmReflect == VMXREFLECTXCPT_NONE);
4692	switch (enmReflect)
4693	{
4694	case VMXREFLECTXCPT_XCPT:
4695	{
4696	Assert(!pVCpu->hm.s.Event.fPending);
4697	pVCpu->hm.s.Event.fPending = true;
4698	pVCpu->hm.s.Event.u64IntrInfo = VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo);
4699	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4700	{
4701	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4702	AssertRCReturn(rc, rc);
4703	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4704	}
4705	else
4706	pVCpu->hm.s.Event.u32ErrCode = 0;
4707	Log(("Pending event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4708	break;
4709	}
4710
4711	case VMXREFLECTXCPT_DF:
4712	{
4713	uint32_t u32IntrInfo;
4714	u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4715	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4716	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4717
4718	Assert(!pVCpu->hm.s.Event.fPending);
4719	pVCpu->hm.s.Event.fPending = true;
4720	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4721	pVCpu->hm.s.Event.u32ErrCode = 0;
4722	rc = VINF_VMX_DOUBLE_FAULT;
4723	Log(("Pending #DF %#RX64 uIdt=%#x uExit=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4724	break;
4725	}
4726
4727	case VMXREFLECTXCPT_TF:
4728	{
4729	Log(("Pending triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4730	rc = VINF_EM_RESET;
4731	break;
4732	}
4733
4734	default: /* shut up gcc. */
4735	break;
4736	}
4737	}
4738	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET \|\| rc == VINF_VMX_DOUBLE_FAULT);
4739	return rc;
4740	}
4741
4742
4743	/**
4744	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4745	*
4746	* @returns VBox status code.
4747	* @param pVM Pointer to the VM.
4748	* @param pVCpu Pointer to the VMCPU.
4749	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4750	* out-of-sync. Make sure to update the required fields
4751	* before using them.
4752	*
4753	* @remarks No-long-jump zone!!!
4754	*/
4755	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4756	{
4757	int rc = VINF_SUCCESS;
4758	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4759	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4760	{
4761	RTGCUINTREG uVal = 0;
4762	RTCCUINTREG uShadow = 0;
4763	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4764	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4765	AssertRCReturn(rc, rc);
4766	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4767	CPUMSetGuestCR0(pVCpu, uVal);
4768	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4769	}
4770	return rc;
4771	}
4772
4773
4774	/**
4775	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4776	*
4777	* @returns VBox status code.
4778	* @param pVM Pointer to the VM.
4779	* @param pVCpu Pointer to the VMCPU.
4780	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4781	* out-of-sync. Make sure to update the required fields
4782	* before using them.
4783	*
4784	* @remarks No-long-jump zone!!!
4785	*/
4786	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4787	{
4788	int rc = VINF_SUCCESS;
4789	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4790	{
4791	RTGCUINTREG uVal = 0;
4792	RTCCUINTREG uShadow = 0;
4793	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4794	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4795	AssertRCReturn(rc, rc);
4796	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4797	CPUMSetGuestCR4(pVCpu, uVal);
4798	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4799	}
4800	return rc;
4801	}
4802
4803
4804	/**
4805	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4806	*
4807	* @returns VBox status code.
4808	* @param pVM Pointer to the VM.
4809	* @param pVCpu Pointer to the VMCPU.
4810	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4811	* out-of-sync. Make sure to update the required fields
4812	* before using them.
4813	*
4814	* @remarks No-long-jump zone!!!
4815	*/
4816	DECLINLINE(int) hmR0VmxSaveGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4817	{
4818	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4819	return VINF_SUCCESS;
4820
4821	RTGCUINTREG uVal = 0;
4822	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4823	AssertRCReturn(rc, rc);
4824	pMixedCtx->rip = uVal;
4825	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4826	return rc;
4827	}
4828
4829
4830	/**
4831	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4832	*
4833	* @returns VBox status code.
4834	* @param pVM Pointer to the VM.
4835	* @param pVCpu Pointer to the VMCPU.
4836	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4837	* out-of-sync. Make sure to update the required fields
4838	* before using them.
4839	*
4840	* @remarks No-long-jump zone!!!
4841	*/
4842	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4843	{
4844	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4845	return VINF_SUCCESS;
4846
4847	RTGCUINTREG uVal = 0;
4848	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4849	AssertRCReturn(rc, rc);
4850	pMixedCtx->rsp = uVal;
4851	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4852	return rc;
4853	}
4854
4855
4856	/**
4857	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4858	*
4859	* @returns VBox status code.
4860	* @param pVM Pointer to the VM.
4861	* @param pVCpu Pointer to the VMCPU.
4862	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4863	* out-of-sync. Make sure to update the required fields
4864	* before using them.
4865	*
4866	* @remarks No-long-jump zone!!!
4867	*/
4868	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4869	{
4870	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4871	return VINF_SUCCESS;
4872
4873	RTGCUINTREG uVal = 0;
4874	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4875	AssertRCReturn(rc, rc);
4876	pMixedCtx->rflags.u64 = uVal;
4877
4878	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4879	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4880	{
4881	Assert(pVM->hm.s.vmx.pRealModeTSS);
4882	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4883	pMixedCtx->eflags.Bits.u1VM = 0;
4884	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4885	}
4886
4887	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4888	return rc;
4889	}
4890
4891
4892	/**
4893	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4894	* guest-CPU context.
4895	*/
4896	static int hmR0VmxSaveGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4897	{
4898	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4899	rc \|= hmR0VmxSaveGuestRsp(pVM, pVCpu, pMixedCtx);
4900	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
4901	return rc;
4902	}
4903
4904
4905	/**
4906	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4907	* from the guest-state area in the VMCS.
4908	*
4909	* @param pVM Pointer to the VM.
4910	* @param pVCpu Pointer to the VMCPU.
4911	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4912	* out-of-sync. Make sure to update the required fields
4913	* before using them.
4914	*
4915	* @remarks No-long-jump zone!!!
4916	*/
4917	DECLINLINE(void) hmR0VmxSaveGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4918	{
4919	uint32_t uIntrState = 0;
4920	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4921	AssertRC(rc);
4922
4923	if (!uIntrState)
4924	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4925	else
4926	{
4927	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4928	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4929	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4930	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* for hmR0VmxLoadGuestIntrState(). */
4931	AssertRC(rc);
4932	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4933	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4934	}
4935	}
4936
4937
4938	/**
4939	* Saves the guest's activity state.
4940	*
4941	* @returns VBox status code.
4942	* @param pVM Pointer to the VM.
4943	* @param pVCpu Pointer to the VMCPU.
4944	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4945	* out-of-sync. Make sure to update the required fields
4946	* before using them.
4947	*
4948	* @remarks No-long-jump zone!!!
4949	*/
4950	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4951	{
4952	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4953	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
4954	return VINF_SUCCESS;
4955	}
4956
4957
4958	/**
4959	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4960	* the current VMCS into the guest-CPU context.
4961	*
4962	* @returns VBox status code.
4963	* @param pVM Pointer to the VM.
4964	* @param pVCpu Pointer to the VMCPU.
4965	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4966	* out-of-sync. Make sure to update the required fields
4967	* before using them.
4968	*
4969	* @remarks No-long-jump zone!!!
4970	*/
4971	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4972	{
4973	int rc = VINF_SUCCESS;
4974	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4975	{
4976	uint32_t u32Val = 0;
4977	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4978	pMixedCtx->SysEnter.cs = u32Val;
4979	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4980	}
4981
4982	RTGCUINTREG uGCVal = 0;
4983	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4984	{
4985	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4986	pMixedCtx->SysEnter.eip = uGCVal;
4987	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4988	}
4989	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4990	{
4991	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4992	pMixedCtx->SysEnter.esp = uGCVal;
4993	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4994	}
4995	return rc;
4996	}
4997
4998
4999	/**
5000	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5001	* context.
5002	*
5003	* @returns VBox status code.
5004	* @param pVM Pointer to the VM.
5005	* @param pVCpu Pointer to the VMCPU.
5006	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5007	* out-of-sync. Make sure to update the required fields
5008	* before using them.
5009	*
5010	* @remarks No-long-jump zone!!!
5011	*/
5012	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5013	{
5014	RTGCUINTREG uVal = 0;
5015	int rc = VINF_SUCCESS;
5016	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
5017	{
5018	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
5019	pMixedCtx->fs.u64Base = uVal;
5020	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
5021	}
5022	return rc;
5023	}
5024
5025
5026	/**
5027	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5028	* context.
5029	*
5030	* @returns VBox status code.
5031	* @param pVM Pointer to the VM.
5032	* @param pVCpu Pointer to the VMCPU.
5033	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5034	* out-of-sync. Make sure to update the required fields
5035	* before using them.
5036	*
5037	* @remarks No-long-jump zone!!!
5038	*/
5039	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5040	{
5041	RTGCUINTREG uVal = 0;
5042	int rc = VINF_SUCCESS;
5043	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
5044	{
5045	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5046	pMixedCtx->gs.u64Base = uVal;
5047	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
5048	}
5049	return rc;
5050	}
5051
5052
5053	/**
5054	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5055	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5056	*
5057	* @returns VBox status code.
5058	* @param pVM Pointer to the VM.
5059	* @param pVCpu Pointer to the VMCPU.
5060	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5061	* out-of-sync. Make sure to update the required fields
5062	* before using them.
5063	*
5064	* @remarks No-long-jump zone!!!
5065	*/
5066	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5067	{
5068	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5069	return VINF_SUCCESS;
5070
5071	for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5072	{
5073	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5074	pMsr += i;
5075	switch (pMsr->u32IndexMSR)
5076	{
5077	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5078	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5079	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5080	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5081	#if 0
5082	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5083	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5084	#endif
5085	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5086	default:
5087	{
5088	AssertFailed();
5089	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5090	}
5091	}
5092	}
5093	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5094	return VINF_SUCCESS;
5095	}
5096
5097
5098	/**
5099	* Saves the guest control registers from the current VMCS into the guest-CPU
5100	* context.
5101	*
5102	* @returns VBox status code.
5103	* @param pVM Pointer to the VM.
5104	* @param pVCpu Pointer to the VMCPU.
5105	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5106	* out-of-sync. Make sure to update the required fields
5107	* before using them.
5108	*
5109	* @remarks No-long-jump zone!!!
5110	*/
5111	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5112	{
5113	RTGCUINTREG uVal = 0;
5114	RTGCUINTREG uShadow = 0;
5115	int rc = VINF_SUCCESS;
5116
5117	/* Guest CR0. Guest FPU. */
5118	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5119
5120	/* Guest CR4. */
5121	rc \|= hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
5122	AssertRCReturn(rc, rc);
5123
5124	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5125	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5126	{
5127	if ( pVM->hm.s.fNestedPaging
5128	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5129	{
5130	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5131	if (pMixedCtx->cr3 != uVal)
5132	{
5133	CPUMSetGuestCR3(pVCpu, uVal);
5134	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5135	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5136	}
5137
5138	/* We require EFER to check PAE mode. */
5139	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5140
5141	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5142	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5143	{
5144	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5145	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5146	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5147	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5148	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5149	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5150	}
5151	AssertRCReturn(rc, rc);
5152	}
5153	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5154	}
5155	return rc;
5156	}
5157
5158
5159	/**
5160	* Reads a guest segment register from the current VMCS into the guest-CPU
5161	* context.
5162	*
5163	* @returns VBox status code.
5164	* @param pVCpu Pointer to the VMCPU.
5165	* @param idxSel Index of the selector in the VMCS.
5166	* @param idxLimit Index of the segment limit in the VMCS.
5167	* @param idxBase Index of the segment base in the VMCS.
5168	* @param idxAccess Index of the access rights of the segment in the VMCS.
5169	* @param pSelReg Pointer to the segment selector.
5170	*
5171	* @remarks No-long-jump zone!!!
5172	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5173	* as that takes care of whether to read from the VMCS cache or not.
5174	*/
5175	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5176	PCPUMSELREG pSelReg)
5177	{
5178	uint32_t u32Val = 0;
5179	int rc = VMXReadVmcs32(idxSel, &u32Val);
5180	pSelReg->Sel = (uint16_t)u32Val;
5181	pSelReg->ValidSel = (uint16_t)u32Val;
5182	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5183
5184	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5185	pSelReg->u32Limit = u32Val;
5186
5187	RTGCUINTREG uGCVal = 0;
5188	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5189	pSelReg->u64Base = uGCVal;
5190
5191	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5192	pSelReg->Attr.u = u32Val;
5193	AssertRCReturn(rc, rc);
5194
5195	/*
5196	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5197	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5198	*/
5199	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5200	{
5201	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5202	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5203	}
5204	return rc;
5205	}
5206
5207
5208	/**
5209	* Saves the guest segment registers from the current VMCS into the guest-CPU
5210	* context.
5211	*
5212	* @returns VBox status code.
5213	* @param pVM Pointer to the VM.
5214	* @param pVCpu Pointer to the VMCPU.
5215	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5216	* out-of-sync. Make sure to update the required fields
5217	* before using them.
5218	*
5219	* @remarks No-long-jump zone!!!
5220	*/
5221	static int hmR0VmxSaveGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5222	{
5223	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5224	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5225	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5226	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5227	#else
5228	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5229	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5230	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5231	#endif
5232
5233	int rc = VINF_SUCCESS;
5234
5235	/* Guest segment registers. */
5236	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5237	{
5238	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5239	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5240	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5241	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5242	rc \|= VMXLOCAL_READ_SEG(ES, es);
5243	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5244	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5245	AssertRCReturn(rc, rc);
5246
5247	/* Restore segment attributes for real-on-v86 mode hack. */
5248	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5249	{
5250	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5251	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5252	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5253	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5254	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5255	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5256	}
5257	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5258	}
5259
5260	/* Guest LDTR. */
5261	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5262	{
5263	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5264	AssertRCReturn(rc, rc);
5265	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5266	}
5267
5268	/* Guest GDTR. */
5269	RTGCUINTREG uGCVal = 0;
5270	uint32_t u32Val = 0;
5271	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5272	{
5273	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5274	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5275	pMixedCtx->gdtr.pGdt = uGCVal;
5276	pMixedCtx->gdtr.cbGdt = u32Val;
5277	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5278	}
5279
5280	/* Guest IDTR. */
5281	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5282	{
5283	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5284	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5285	pMixedCtx->idtr.pIdt = uGCVal;
5286	pMixedCtx->idtr.cbIdt = u32Val;
5287	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5288	}
5289
5290	/* Guest TR. */
5291	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5292	{
5293	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5294
5295	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5296	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5297	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5298	AssertRCReturn(rc, rc);
5299	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5300	}
5301	return rc;
5302	}
5303
5304
5305	/**
5306	* Saves the guest debug registers from the current VMCS into the guest-CPU
5307	* context.
5308	*
5309	* @returns VBox status code.
5310	* @param pVM Pointer to the VM.
5311	* @param pVCpu Pointer to the VMCPU.
5312	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5313	* out-of-sync. Make sure to update the required fields
5314	* before using them.
5315	*
5316	* @remarks No-long-jump zone!!!
5317	*/
5318	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5319	{
5320	int rc = VINF_SUCCESS;
5321	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5322	{
5323	RTGCUINTREG uVal;
5324	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5325	pMixedCtx->dr[7] = uVal;
5326
5327	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5328	}
5329	return rc;
5330	}
5331
5332
5333	/**
5334	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5335	*
5336	* @returns VBox status code.
5337	* @param pVM Pointer to the VM.
5338	* @param pVCpu Pointer to the VMCPU.
5339	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5340	* out-of-sync. Make sure to update the required fields
5341	* before using them.
5342	*
5343	* @remarks No-long-jump zone!!!
5344	*/
5345	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5346	{
5347	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5348	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5349	return VINF_SUCCESS;
5350	}
5351
5352
5353	/**
5354	* Saves the entire guest state from the currently active VMCS into the
5355	* guest-CPU context. This essentially VMREADs all guest-data.
5356	*
5357	* @returns VBox status code.
5358	* @param pVM Pointer to the VM.
5359	* @param pVCpu Pointer to the VMCPU.
5360	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5361	* out-of-sync. Make sure to update the required fields
5362	* before using them.
5363	*/
5364	static int hmR0VmxSaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5365	{
5366	Assert(pVM);
5367	Assert(pVCpu);
5368	Assert(pMixedCtx);
5369
5370	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5371	return VINF_SUCCESS;
5372
5373	VMMRZCallRing3Disable(pVCpu);
5374
5375	int rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
5376	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGprs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5377
5378	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5379	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5380
5381	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5382	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5383
5384	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx);
5385	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5386
5387	rc = hmR0VmxSaveGuestSysenterMsrs(pVM, pVCpu, pMixedCtx);
5388	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5389
5390	rc = hmR0VmxSaveGuestFSBaseMsr(pVM, pVCpu, pMixedCtx);
5391	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5392
5393	rc = hmR0VmxSaveGuestGSBaseMsr(pVM, pVCpu, pMixedCtx);
5394	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5395
5396	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5397	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5398
5399	rc = hmR0VmxSaveGuestActivityState(pVM, pVCpu, pMixedCtx);
5400	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5401
5402	rc = hmR0VmxSaveGuestApicState(pVM, pVCpu, pMixedCtx);
5403	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5404
5405	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5406	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5407
5408	VMMRZCallRing3Enable(pVCpu);
5409	return rc;
5410	}
5411
5412
5413	/**
5414	* Check per-VM and per-VCPU force flag actions that require us to go back to
5415	* ring-3 for one reason or another.
5416	*
5417	* @returns VBox status code (information status code included).
5418	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5419	* ring-3.
5420	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5421	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5422	* interrupts)
5423	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5424	* all EMTs to be in ring-3.
5425	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5426	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5427	* to the EM loop.
5428	*
5429	* @param pVM Pointer to the VM.
5430	* @param pVCpu Pointer to the VMCPU.
5431	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5432	* out-of-sync. Make sure to update the required fields
5433	* before using them.
5434	*/
5435	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5436	{
5437	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5438
5439	int rc = VERR_INTERNAL_ERROR_5;
5440	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5441	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5442	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5443	{
5444	/* We need the control registers now, make sure the guest-CPU context is updated. */
5445	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5446	AssertRCReturn(rc, rc);
5447
5448	/* Pending HM CR3 sync. */
5449	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5450	{
5451	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5452	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5453	}
5454	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5455	{
5456	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5457	AssertRC(rc);
5458	}
5459
5460	/* Pending PGM C3 sync. */
5461	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5462	{
5463	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5464	if (rc != VINF_SUCCESS)
5465	{
5466	AssertRC(rc);
5467	Log2(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5468	return rc;
5469	}
5470	}
5471
5472	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5473	/* -XXX- what was that about single stepping? */
5474	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5475	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5476	{
5477	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5478	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5479	Log2(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5480	return rc;
5481	}
5482
5483	/* Pending VM request packets, such as hardware interrupts. */
5484	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5485	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5486	{
5487	Log2(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5488	return VINF_EM_PENDING_REQUEST;
5489	}
5490
5491	/* Pending PGM pool flushes. */
5492	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5493	{
5494	Log2(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5495	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5496	}
5497
5498	/* Pending DMA requests. */
5499	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5500	{
5501	Log2(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5502	return VINF_EM_RAW_TO_R3;
5503	}
5504	}
5505
5506	/* Paranoia. */
5507	Assert(rc != VERR_EM_INTERPRETER);
5508	return VINF_SUCCESS;
5509	}
5510
5511
5512	/**
5513	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5514	* VT-x to execute any instruction.
5515	*
5516	* @param pvCpu Pointer to the VMCPU.
5517	*/
5518	static void hmR0VmxUpdateTRPMTrap(PVMCPU pVCpu)
5519	{
5520	if (pVCpu->hm.s.Event.fPending)
5521	{
5522	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5523	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5524	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5525	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5526
5527	/* If a trap was already pending, we did something wrong! */
5528	Assert(TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP);
5529
5530	/* A page-fault exception during a page-fault would become a double-fault. */
5531	AssertMsg(uVectorType != VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT \|\| uVector != X86_XCPT_PF,
5532	("%#RX64 uVectorType=%#x uVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uVectorType, uVector));
5533
5534	TRPMEVENT enmTrapType;
5535	switch (uVectorType)
5536	{
5537	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5538	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5539	enmTrapType = TRPM_HARDWARE_INT;
5540	break;
5541	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5542	enmTrapType = TRPM_SOFTWARE_INT;
5543	break;
5544	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5545	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5546	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5547	enmTrapType = TRPM_TRAP;
5548	break;
5549	default:
5550	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5551	enmTrapType = TRPM_32BIT_HACK;
5552	break;
5553	}
5554	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5555	AssertRC(rc);
5556	if (fErrorCodeValid)
5557	TRPMSetErrorCode(pVCpu, uErrorCode);
5558
5559	/* Clear the VT-x state bits now that TRPM has the information. */
5560	pVCpu->hm.s.Event.fPending = false;
5561	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
5562	AssertRC(rc);
5563	}
5564	}
5565
5566
5567	/**
5568	* Does the necessary state syncing before doing a longjmp to ring-3.
5569	*
5570	* @param pVM Pointer to the VM.
5571	* @param pVCpu Pointer to the VMCPU.
5572	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5573	* out-of-sync. Make sure to update the required fields
5574	* before using them.
5575	* @param rcExit The reason for exiting to ring-3. Can be
5576	* VINF_VMM_UNKNOWN_RING3_CALL.
5577	*
5578	* @remarks No-long-jmp zone!!!
5579	*/
5580	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5581	{
5582	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5583
5584	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
5585	AssertRC(rc);
5586
5587	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5588	if (CPUMIsGuestFPUStateActive(pVCpu))
5589	{
5590	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5591	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5592	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5593	}
5594
5595	/* Restore debug registers if necessary and resync on next R0 reentry. */
5596	if (CPUMIsGuestDebugStateActive(pVCpu))
5597	{
5598	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5599	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5600	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5601	}
5602	else if (CPUMIsHyperDebugStateActive(pVCpu))
5603	{
5604	CPUMR0LoadHostDebugState(pVM, pVCpu);
5605	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5606	}
5607
5608	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5609	}
5610
5611
5612	/**
5613	* An action requires us to go back to ring-3. This function does the necessary
5614	* steps before we can safely return to ring-3. This is not the same as longjmps
5615	* to ring-3, this is voluntary.
5616	*
5617	* @param pVM Pointer to the VM.
5618	* @param pVCpu Pointer to the VMCPU.
5619	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5620	* out-of-sync. Make sure to update the required fields
5621	* before using them.
5622	* @param rcExit The reason for exiting to ring-3. Can be
5623	* VINF_VMM_UNKNOWN_RING3_CALL.
5624	*/
5625	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5626	{
5627	Assert(pVM);
5628	Assert(pVCpu);
5629	Assert(pMixedCtx);
5630	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5631
5632	/* We want to see what the guest-state was before VM-entry, don't resync here, as we will never continue guest execution.*/
5633	if (rcExit == VERR_VMX_INVALID_GUEST_STATE)
5634	return;
5635
5636	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5637	VMMRZCallRing3Disable(pVCpu);
5638	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5639
5640	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5641	hmR0VmxUpdateTRPMTrap(pVCpu);
5642
5643	/* Sync. the guest state. */
5644	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5645	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5646
5647	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5648	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5649	\| CPUM_CHANGED_LDTR
5650	\| CPUM_CHANGED_GDTR
5651	\| CPUM_CHANGED_IDTR
5652	\| CPUM_CHANGED_TR
5653	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5654
5655	/* On our way back from ring-3 the following needs to be done. */
5656	/** @todo This can change with preemption hooks. */
5657	if (rcExit == VINF_EM_RAW_INTERRUPT)
5658	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5659	else
5660	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5661
5662	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5663	VMMRZCallRing3Enable(pVCpu);
5664	}
5665
5666
5667	/**
5668	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5669	* longjump to ring-3 and possibly get preempted.
5670	*
5671	* @param pVCpu Pointer to the VMCPU.
5672	* @param enmOperation The operation causing the ring-3 longjump.
5673	* @param pvUser The user argument (pointer to the possibly
5674	* out-of-date guest-CPU context).
5675	*
5676	* @remarks Must never be called with @a enmOperation ==
5677	* VMMCALLRING3_VM_R0_ASSERTION.
5678	*/
5679	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5680	{
5681	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5682	Assert(pVCpu);
5683	Assert(pvUser);
5684	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5685	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5686
5687	VMMRZCallRing3Disable(pVCpu);
5688	Log(("hmR0VmxLongJmpToRing3\n"));
5689	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5690	VMMRZCallRing3Enable(pVCpu);
5691	}
5692
5693
5694	/**
5695	* Injects any pending TRPM trap into the VM by updating the VMCS.
5696	*
5697	* @returns VBox status code (informational status code included).
5698	* @param pVM Pointer to the VM.
5699	* @param pVCpu Pointer to the VMCPU.
5700	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5701	* out-of-sync. Make sure to update the required fields
5702	* before using them.
5703	*/
5704	static int hmR0VmxInjectTRPMTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5705	{
5706	if (!TRPMHasTrap(pVCpu))
5707	return VINF_SUCCESS;
5708
5709	uint8_t u8Vector = 0;
5710	TRPMEVENT enmTrpmEvent = TRPM_SOFTWARE_INT;
5711	RTGCUINT uErrCode = 0;
5712
5713	int rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmTrpmEvent, &uErrCode, NULL /* puCr2 */);
5714	AssertRCReturn(rc, rc);
5715	Assert(enmTrpmEvent != TRPM_SOFTWARE_INT);
5716
5717	rc = TRPMResetTrap(pVCpu);
5718	AssertRCReturn(rc, rc);
5719
5720	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5721	uint32_t u32IntrInfo = u8Vector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5722	if (enmTrpmEvent == TRPM_TRAP)
5723	{
5724	switch (u8Vector)
5725	{
5726	case X86_XCPT_BP:
5727	case X86_XCPT_OF:
5728	{
5729	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5730	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5731	break;
5732	}
5733
5734	case X86_XCPT_DF:
5735	case X86_XCPT_TS:
5736	case X86_XCPT_NP:
5737	case X86_XCPT_SS:
5738	case X86_XCPT_GP:
5739	case X86_XCPT_PF:
5740	case X86_XCPT_AC:
5741	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5742	them which VT-x/VMM pushes to the guest stack. */
5743	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5744	/* no break! */
5745	default:
5746	{
5747	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5748	break;
5749	}
5750	}
5751	}
5752	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5753	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5754	else
5755	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5756
5757	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5758	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, uErrCode);
5759	}
5760
5761
5762	/**
5763	* Checks if there are any pending guest interrupts to be delivered and injects
5764	* them into the VM by updating the VMCS.
5765	*
5766	* @returns VBox status code (informational status codes included).
5767	* @param pVM Pointer to the VM.
5768	* @param pVCpu Pointer to the VMCPU.
5769	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5770	* out-of-sync. Make sure to update the required fields
5771	* before using them.
5772	*
5773	* @remarks Must be called after hmR0VmxLoadGuestIntrState().
5774	*/
5775	static int hmR0VmxInjectPendingInterrupt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5776	{
5777	/* First inject any pending HM interrupts. */
5778	if (pVCpu->hm.s.Event.fPending)
5779	{
5780	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, 0 /* cbInstr */,
5781	pVCpu->hm.s.Event.u32ErrCode);
5782	AssertRCReturn(rc, rc);
5783	pVCpu->hm.s.Event.fPending = false;
5784	return rc;
5785	}
5786
5787	/** @todo SMI. SMIs take priority over NMIs. */
5788
5789	/* NMI. NMIs take priority over regular interrupts . */
5790	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
5791	{
5792	/* Construct an NMI interrupt and inject it into the VMCS. */
5793	RTGCUINTPTR uIntrInfo;
5794	uIntrInfo = X86_XCPT_NMI;
5795	uIntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5796	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5797	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5798	AssertRCReturn(rc, rc);
5799	return rc;
5800	}
5801
5802	/* We need the guests's RFLAGS for sure from this point on, make sure it is updated. */
5803	int rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
5804	AssertRCReturn(rc, rc);
5805
5806	/* If there isn't any active trap, check if we have pending interrupts and convert them to TRPM traps and deliver them. */
5807	if (!TRPMHasTrap(pVCpu))
5808	{
5809	/* Check if there are guest external interrupts (PIC/APIC) pending. */
5810	if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5811	{
5812	/*
5813	* If the guest can receive interrupts now (interrupts enabled and no interrupt inhibition is active) convert
5814	* the PDM interrupt into a TRPM event and inject it.
5815	*/
5816	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5817	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5818	{
5819	uint8_t u8Interrupt = 0;
5820	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5821	if (RT_SUCCESS(rc))
5822	{
5823	/* Convert pending interrupt from PIC/APIC into TRPM and handle it below. */
5824	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
5825	AssertRCReturn(rc, rc);
5826	}
5827	else
5828	{
5829	/** @todo Does this actually happen? If not turn it into an assertion. */
5830	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5831	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5832	}
5833	}
5834	else if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5835	{
5836	/* Instruct VT-x to cause an interrupt-window exit as soon as the guest is ready to receive interrupts again. */
5837	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5838	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5839	AssertRCReturn(rc, rc);
5840	}
5841	/* else we will deliver interrupts whenever the guest exits next and it's in a state to receive interrupts. */
5842	}
5843	}
5844
5845	/* If interrupts can be delivered, inject it into the VM. */
5846	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5847	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
5848	&& TRPMHasTrap(pVCpu))
5849	{
5850	rc = hmR0VmxInjectTRPMTrap(pVM, pVCpu, pMixedCtx);
5851	AssertRCReturn(rc, rc);
5852	}
5853	return rc;
5854	}
5855
5856	/**
5857	* Injects an invalid-opcode (#UD) exception into the VM.
5858	*
5859	* @returns VBox status code (informational status code included).
5860	* @param pVM Pointer to the VM.
5861	* @param pVCpu Pointer to the VMCPU.
5862	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5863	* out-of-sync. Make sure to update the required fields
5864	* before using them.
5865	*/
5866	DECLINLINE(int) hmR0VmxInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5867	{
5868	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5869	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5870	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5871	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5872	}
5873
5874
5875	/**
5876	* Injects a double-fault (#DF) exception into the VM.
5877	*
5878	* @returns VBox status code (informational status code included).
5879	* @param pVM Pointer to the VM.
5880	* @param pVCpu Pointer to the VMCPU.
5881	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5882	* out-of-sync. Make sure to update the required fields
5883	* before using them.
5884	*/
5885	DECLINLINE(int) hmR0VmxInjectXcptDF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5886	{
5887	/* Inject the double-fault. */
5888	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5889	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5890	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5891	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5892	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5893	}
5894
5895
5896	/**
5897	* Injects a debug (#DB) exception into the VM.
5898	*
5899	* @returns VBox status code (informational status code included).
5900	* @param pVM Pointer to the VM.
5901	* @param pVCpu Pointer to the VMCPU.
5902	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5903	* out-of-sync. Make sure to update the required fields
5904	* before using them.
5905	*/
5906	DECLINLINE(int) hmR0VmxInjectXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5907	{
5908	/* Inject the debug-exception. */
5909	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5910	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5911	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5912	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5913	}
5914
5915
5916	/**
5917	* Injects a overflow (#OF) exception into the VM.
5918	*
5919	* @returns VBox status code (informational status code included).
5920	* @param pVM Pointer to the VM.
5921	* @param pVCpu Pointer to the VMCPU.
5922	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5923	* out-of-sync. Make sure to update the required fields
5924	* before using them.
5925	* @param cbInstr The value of RIP that is to be pushed on the guest
5926	* stack.
5927	*/
5928	DECLINLINE(int) hmR0VmxInjectXcptOF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5929	{
5930	/* Inject the overflow exception. */
5931	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5932	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5933	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5934	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5935	}
5936
5937
5938	/**
5939	* Injects a general-protection (#GP) fault into the VM.
5940	*
5941	* @returns VBox status code (informational status code included).
5942	* @param pVM Pointer to the VM.
5943	* @param pVCpu Pointer to the VMCPU.
5944	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5945	* out-of-sync. Make sure to update the required fields
5946	* before using them.
5947	* @param u32ErrorCode The error code associated with the #GP.
5948	*/
5949	DECLINLINE(int) hmR0VmxInjectXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode)
5950	{
5951	/* Inject the general-protection fault. */
5952	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5953	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5954	if (fErrorCodeValid)
5955	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5956	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5957	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode);
5958	}
5959
5960
5961	/**
5962	* Injects a software interrupt (INTn) into the VM.
5963	*
5964	* @returns VBox status code (informational status code included).
5965	* @param pVM Pointer to the VM.
5966	* @param pVCpu Pointer to the VMCPU.
5967	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5968	* out-of-sync. Make sure to update the required fields
5969	* before using them.
5970	* @param uVector The software interrupt vector number.
5971	* @param cbInstr The value of RIP that is to be pushed on the guest
5972	* stack.
5973	*/
5974	DECLINLINE(int) hmR0VmxInjectIntN(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
5975	{
5976	/* Inject the INTn. */
5977	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5978	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5979	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5980	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5981	}
5982
5983
5984	/**
5985	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
5986	* stack.
5987	*
5988	* @returns VBox status code (information status code included).
5989	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
5990	* @param pVM Pointer to the VM.
5991	* @param pMixedCtx Pointer to the guest-CPU context.
5992	* @param uValue The value to push to the guest stack.
5993	*/
5994	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
5995	{
5996	/*
5997	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
5998	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5999	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6000	*/
6001	if (pMixedCtx->sp == 1)
6002	return VINF_EM_RESET;
6003	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6004	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6005	AssertRCReturn(rc, rc);
6006	return rc;
6007	}
6008
6009
6010	/**
6011	* Injects an event into the guest upon VM-entry by updating the relevant fields
6012	* in the VM-entry area in the VMCS.
6013	*
6014	* @returns VBox status code (informational error codes included).
6015	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6016	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6017	*
6018	* @param pVM Pointer to the VM.
6019	* @param pVCpu Pointer to the VMCPU.
6020	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6021	* out-of-sync. Make sure to update the required fields
6022	* before using them.
6023	* @param u64IntrInfo The VM-entry interruption-information field.
6024	* @param cbInstr The VM-entry instruction length in bytes (for software
6025	* interrupts, exceptions and privileged software
6026	* exceptions).
6027	* @param u32ErrCode The VM-entry exception error code.
6028	*
6029	* @remarks No-long-jump zone!!!
6030	*/
6031	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6032	uint32_t u32ErrCode)
6033	{
6034	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6035	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6036	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6037
6038	/* We require CR0 to check if the guest is in real-mode. */
6039	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6040	AssertRCReturn(rc, rc);
6041
6042	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6043	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6044
6045	/*
6046	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6047	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6048	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6049	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6050	*/
6051	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6052	{
6053	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6054	{
6055	Assert(PDMVmmDevHeapIsEnabled(pVM));
6056	Assert(pVM->hm.s.vmx.pRealModeTSS);
6057
6058	/* Save the required guest state bits from the VMCS. */
6059	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
6060	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
6061	AssertRCReturn(rc, rc);
6062
6063	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6064	const size_t cbIdtEntry = 4;
6065	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6066	{
6067	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6068	if (uVector == X86_XCPT_DF)
6069	return VINF_EM_RESET;
6070	else if (uVector == X86_XCPT_GP)
6071	{
6072	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6073	return hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
6074	}
6075
6076	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6077	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6078	return hmR0VmxInjectXcptGP(pVM, pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */);
6079	}
6080
6081	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6082	uint16_t uGuestIp = pMixedCtx->ip;
6083	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6084	{
6085	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6086	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6087	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6088	}
6089	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6090	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6091
6092	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6093	uint16_t offIdtEntry = 0;
6094	RTSEL selIdtEntry = 0;
6095	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6096	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6097	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6098	AssertRCReturn(rc, rc);
6099
6100	/* Construct the stack frame for the interrupt/exception handler. */
6101	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6102	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6103	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6104	AssertRCReturn(rc, rc);
6105
6106	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6107	if (rc == VINF_SUCCESS)
6108	{
6109	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6110	pMixedCtx->rip = offIdtEntry;
6111	pMixedCtx->cs.Sel = selIdtEntry;
6112	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6113	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6114	\| HM_CHANGED_GUEST_RIP
6115	\| HM_CHANGED_GUEST_RFLAGS
6116	\| HM_CHANGED_GUEST_RSP;
6117	}
6118	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6119	return rc;
6120	}
6121	else
6122	{
6123	/*
6124	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6125	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6126	*/
6127	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6128	}
6129	}
6130
6131	/* Add the valid bit, maybe the caller was lazy. */
6132	u32IntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6133
6134	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6135	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6136	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6137
6138	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6139	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6140	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6141	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6142	AssertRCReturn(rc, rc);
6143	return rc;
6144	}
6145
6146
6147	/**
6148	* Enters the VT-x session.
6149	*
6150	* @returns VBox status code.
6151	* @param pVM Pointer to the VM.
6152	* @param pVCpu Pointer to the VMCPU.
6153	* @param pCpu Pointer to the CPU info struct.
6154	*/
6155	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6156	{
6157	AssertPtr(pVM);
6158	AssertPtr(pVCpu);
6159	Assert(pVM->hm.s.vmx.fSupported);
6160	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6161	NOREF(pCpu);
6162
6163	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6164
6165	/* Make sure we're in VMX root mode. */
6166	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6167	if (!(u32HostCR4 & X86_CR4_VMXE))
6168	{
6169	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6170	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6171	}
6172
6173	/* Load the active VMCS as the current one. */
6174	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6175	if (RT_FAILURE(rc))
6176	return rc;
6177
6178	/** @todo this will change with preemption hooks where can can VMRESUME as long
6179	* as we're no preempted. */
6180	pVCpu->hm.s.fResumeVM = false;
6181	return VINF_SUCCESS;
6182	}
6183
6184
6185	/**
6186	* Leaves the VT-x session.
6187	*
6188	* @returns VBox status code.
6189	* @param pVM Pointer to the VM.
6190	* @param pVCpu Pointer to the VMCPU.
6191	* @param pCtx Pointer to the guest-CPU context.
6192	*/
6193	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6194	{
6195	AssertPtr(pVCpu);
6196	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6197	NOREF(pVM);
6198	NOREF(pCtx);
6199
6200	/** @todo this will change with preemption hooks where we only VMCLEAR when
6201	* we are actually going to be preempted, not all the time like we
6202	* currently do. */
6203	/*
6204	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6205	* and mark the VMCS launch-state as "clear".
6206	*/
6207	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6208	return rc;
6209	}
6210
6211
6212	/**
6213	* Saves the host state in the VMCS host-state.
6214	* Sets up the VM-exit MSR-load area.
6215	*
6216	* The CPU state will be loaded from these fields on every successful VM-exit.
6217	*
6218	* @returns VBox status code.
6219	* @param pVM Pointer to the VM.
6220	* @param pVCpu Pointer to the VMCPU.
6221	*
6222	* @remarks No-long-jump zone!!!
6223	*/
6224	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6225	{
6226	AssertPtr(pVM);
6227	AssertPtr(pVCpu);
6228	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6229
6230	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6231
6232	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6233	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6234	return VINF_SUCCESS;
6235
6236	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6237	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6238
6239	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6240	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6241
6242	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6243	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6244
6245	/* Reset the host-state-changed flag. */
6246	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6247	return rc;
6248	}
6249
6250
6251	/**
6252	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6253	* loaded from these fields on every successful VM-entry.
6254	*
6255	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6256	* Sets up the VM-entry controls.
6257	* Sets up the appropriate VMX non-root function to execute guest code based on
6258	* the guest CPU mode.
6259	*
6260	* @returns VBox status code.
6261	* @param pVM Pointer to the VM.
6262	* @param pVCpu Pointer to the VMCPU.
6263	* @param pCtx Pointer to the guest-CPU context.
6264	*
6265	* @remarks No-long-jump zone!!!
6266	*/
6267	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6268	{
6269	AssertPtr(pVM);
6270	AssertPtr(pVCpu);
6271	AssertPtr(pCtx);
6272	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6273
6274	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6275
6276	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6277
6278	/* Determine real-on-v86 mode. */
6279	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6280	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6281	&& CPUMIsGuestInRealModeEx(pCtx))
6282	{
6283	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6284	}
6285
6286	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6287	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6288
6289	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu, pCtx);
6290	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6291
6292	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6293	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6294
6295	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6296	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6297
6298	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6299	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6300
6301	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6302	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6303
6304	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6305	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6306
6307	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6308	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6309
6310	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6311	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6312
6313	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6314	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6315
6316	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6317	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6318	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6319
6320	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6321	return rc;
6322	}
6323
6324
6325	/**
6326	* Does the preparations before executing guest code in VT-x.
6327	*
6328	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6329	* recompiler. We must be cautious what we do here regarding committing
6330	* guest-state information into the the VMCS assuming we assuredly execute the
6331	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6332	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6333	* that the recompiler can (and should) use them when it resumes guest
6334	* execution. Otherwise such operations must be done when we can no longer
6335	* exit to ring-3.
6336	*
6337	* @returns VBox status code (informational status codes included).
6338	* @retval VINF_SUCCESS if we can proceed with running the guest.
6339	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6340	* into the guest.
6341	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6342	*
6343	* @param pVM Pointer to the VM.
6344	* @param pVCpu Pointer to the VMCPU.
6345	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6346	* out-of-sync. Make sure to update the required fields
6347	* before using them.
6348	* @param pVmxTransient Pointer to the VMX transient structure.
6349	*
6350	* @remarks Called with preemption disabled.
6351	*/
6352	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6353	{
6354	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6355
6356	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6357	PGMRZDynMapFlushAutoSet(pVCpu);
6358	#endif
6359
6360	/* Check force flag actions that might require us to go back to ring-3. */
6361	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6362	if (rc != VINF_SUCCESS)
6363	return rc;
6364
6365	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6366	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6367	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6368	{
6369	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6370	RTGCPHYS GCPhysApicBase;
6371	GCPhysApicBase = pMixedCtx->msrApicBase;
6372	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6373
6374	/* Unalias any existing mapping. */
6375	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6376	AssertRCReturn(rc, rc);
6377
6378	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6379	Log2(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6380	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6381	AssertRCReturn(rc, rc);
6382
6383	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6384	}
6385
6386	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6387	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6388	pVmxTransient->uEFlags = ASMIntDisableFlags();
6389	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6390	{
6391	ASMSetFlags(pVmxTransient->uEFlags);
6392	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6393	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6394	return VINF_EM_RAW_INTERRUPT;
6395	}
6396	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6397	#endif
6398
6399	/*
6400	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6401	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6402	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6403	*/
6404	hmR0VmxLoadGuestIntrState(pVM, pVCpu, pMixedCtx);
6405	rc = hmR0VmxInjectPendingInterrupt(pVM, pVCpu, pMixedCtx);
6406	AssertRCReturn(rc, rc);
6407	return rc;
6408	}
6409
6410
6411	/**
6412	* Prepares to run guest code in VT-x and we've committed to doing so. This
6413	* means there is no backing out to ring-3 or anywhere else at this
6414	* point.
6415	*
6416	* @param pVM Pointer to the VM.
6417	* @param pVCpu Pointer to the VMCPU.
6418	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6419	* out-of-sync. Make sure to update the required fields
6420	* before using them.
6421	* @param pVmxTransient Pointer to the VMX transient structure.
6422	*
6423	* @remarks Called with preemption disabled.
6424	* @remarks No-long-jump zone!!!
6425	*/
6426	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6427	{
6428	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6429	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6430
6431	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6432	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6433	pVmxTransient->uEFlags = ASMIntDisableFlags();
6434	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6435	#endif
6436
6437	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6438	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6439	Log(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6440	int rc = VINF_SUCCESS;
6441	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6442	{
6443	rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pMixedCtx);
6444	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6445	}
6446	else if (pVCpu->hm.s.fContextUseFlags)
6447	{
6448	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6449	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6450	}
6451	AssertRC(rc);
6452	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6453
6454	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6455	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6456	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6457
6458	Assert(pVM->hm.s.vmx.pfnFlushTaggedTlb);
6459	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6460	pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu); /* Flush the TLB of guest entries as necessary. */
6461
6462	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6463	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6464	{
6465	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6466	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6467	}
6468
6469	/*
6470	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6471	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6472	*/
6473	if (pVM->hm.s.fTPRPatchingActive)
6474	{
6475	Assert(!CPUMIsGuestInLongMode(pVCpu));
6476
6477	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6478	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6479	AssertRC(rc);
6480
6481	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6482	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6483	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6484	see hmR0VmxLoadGuestApicState(). */
6485	}
6486
6487	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6488	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6489	to start executing. */
6490	}
6491
6492
6493	/**
6494	* Performs some essential restoration of state after running guest code in
6495	* VT-x.
6496	*
6497	* @param pVM Pointer to the VM.
6498	* @param pVCpu Pointer to the VMCPU.
6499	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6500	* out-of-sync. Make sure to update the required fields
6501	* before using them.
6502	* @param pVmxTransient Pointer to the VMX transient structure.
6503	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6504	*
6505	* @remarks Called with interrupts disabled.
6506	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6507	* unconditionally when it is safe to do so.
6508	*/
6509	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6510	{
6511	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6512	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6513
6514	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6515	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6516	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6517	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6518	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6519
6520	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6521	{
6522	/** @todo Find a way to fix hardcoding a guestimate. */
6523	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6524	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6525	}
6526
6527	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6528	Assert(!(ASMGetFlags() & X86_EFL_IF));
6529	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6530
6531	/* Restore the effects of TPR patching if any. */
6532	if (pVM->hm.s.fTPRPatchingActive)
6533	{
6534	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6535	AssertRC(rc);
6536	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6537	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6538	}
6539
6540	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6541	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6542
6543	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6544	uint32_t uExitReason;
6545	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6546	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6547	AssertRC(rc);
6548	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6549	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6550
6551	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6552	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6553
6554	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6555	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6556	{
6557	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6558	return;
6559	}
6560
6561	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6562	{
6563	/* Update the guest interruptibility-state from the VMCS. */
6564	hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
6565
6566	/*
6567	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6568	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6569	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6570	*/
6571	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6572	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6573	{
6574	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6575	AssertRC(rc);
6576	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6577	}
6578	}
6579	}
6580
6581
6582	/**
6583	* Runs the guest code using VT-x.
6584	*
6585	* @returns VBox status code.
6586	* @param pVM Pointer to the VM.
6587	* @param pVCpu Pointer to the VMCPU.
6588	* @param pCtx Pointer to the guest-CPU context.
6589	*
6590	* @remarks Called with preemption disabled.
6591	*/
6592	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6593	{
6594	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6595	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6596
6597	VMXTRANSIENT VmxTransient;
6598	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6599	int rc = VERR_INTERNAL_ERROR_5;
6600	unsigned cLoops = 0;
6601
6602	for (;; cLoops++)
6603	{
6604	Assert(!HMR0SuspendPending());
6605	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6606	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6607	(unsigned)RTMpCpuId(), cLoops));
6608
6609	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6610	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6611	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6612	if (rc != VINF_SUCCESS)
6613	break;
6614
6615	/*
6616	* No longjmps to ring-3 from this point on!!!
6617	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6618	* This also disables flushing of the R0-logger instance (if any).
6619	*/
6620	VMMRZCallRing3Disable(pVCpu);
6621	VMMRZCallRing3RemoveNotification(pVCpu);
6622	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6623
6624	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6625	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6626
6627	/*
6628	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6629	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6630	*/
6631	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6632	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6633	{
6634	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6635	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6636	return rc;
6637	}
6638
6639	/* Handle the VM-exit. */
6640	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6641	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6642	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6643	rc = (*s_apfnVMExitHandlers[VmxTransient.uExitReason])(pVM, pVCpu, pCtx, &VmxTransient);
6644	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6645	if (rc != VINF_SUCCESS)
6646	break;
6647	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6648	{
6649	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6650	rc = VINF_EM_RAW_INTERRUPT;
6651	break;
6652	}
6653	}
6654
6655	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6656	if (rc == VERR_EM_INTERPRETER)
6657	rc = VINF_EM_RAW_EMULATE_INSTR;
6658	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6659	return rc;
6660	}
6661
6662	#if 0
6663	DECLINLINE(int) hmR0VmxHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, unsigned rcReason)
6664	{
6665	int rc;
6666	switch (rcReason)
6667	{
6668	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6669	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6670	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6671	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6672	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6673	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6674	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6675	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6676	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6677	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6678	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6679	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6680	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6681	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6682	case VMX_EXIT_PREEMPTION_TIMER: rc = hmR0VmxExitPreemptionTimer(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6683	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6684	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6685	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6686	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6687	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6688	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6689	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6690	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6691	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6692	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6693	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6694	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6695	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6696	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6697	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6698	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6699	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6700	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6701
6702	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6703	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6704	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6705	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6706	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6707	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6708	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6709	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6710	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6711
6712	case VMX_EXIT_VMCALL:
6713	case VMX_EXIT_VMCLEAR:
6714	case VMX_EXIT_VMLAUNCH:
6715	case VMX_EXIT_VMPTRLD:
6716	case VMX_EXIT_VMPTRST:
6717	case VMX_EXIT_VMREAD:
6718	case VMX_EXIT_VMRESUME:
6719	case VMX_EXIT_VMWRITE:
6720	case VMX_EXIT_VMXOFF:
6721	case VMX_EXIT_VMXON:
6722	case VMX_EXIT_INVEPT:
6723	case VMX_EXIT_INVVPID:
6724	case VMX_EXIT_VMFUNC:
6725	rc = hmR0VmxExitInjectXcptUD(pVM, pVCpu, pMixedCtx, pVmxTransient);
6726	break;
6727	default:
6728	rc = hmR0VmxExitErrUndefined(pVM, pVCpu, pMixedCtx, pVmxTransient);
6729	break;
6730	}
6731	return rc;
6732	}
6733	#endif
6734
6735	#ifdef DEBUG
6736	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6737	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6738	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6739	# define VMX_ASSERT_PREEMPT_CPUID() \
6740	do \
6741	{ \
6742	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6743	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6744	} while (0)
6745
6746	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6747	do { \
6748	AssertPtr(pVM); \
6749	AssertPtr(pVCpu); \
6750	AssertPtr(pMixedCtx); \
6751	AssertPtr(pVmxTransient); \
6752	Assert(pVmxTransient->fVMEntryFailed == false); \
6753	Assert(ASMIntAreEnabled()); \
6754	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6755	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6756	LogFunc(("\n")); \
6757	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6758	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6759	VMX_ASSERT_PREEMPT_CPUID(); \
6760	} while (0)
6761	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6762	do { \
6763	LogFunc(("\n")); \
6764	} while(0)
6765	#else /* Release builds */
6766	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6767	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6768	#endif
6769
6770
6771	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6772	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6773	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6774	/**
6775	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6776	*/
6777	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6778	{
6779	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6780	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6781	return VINF_SUCCESS;
6782	}
6783
6784
6785	/**
6786	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6787	*/
6788	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6789	{
6790	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6791	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6792	AssertRCReturn(rc, rc);
6793
6794	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6795	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6796	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6797
6798	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6799	return VINF_EM_RAW_INTERRUPT;
6800
6801	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6802	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
6803	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6804	return VINF_SUCCESS;
6805	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6806	return rc;
6807
6808	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6809	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6810	switch (uIntrType)
6811	{
6812	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6813	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6814	/* no break */
6815	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
6816	{
6817	switch (uVector)
6818	{
6819	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6820	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6821	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6822	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6823	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6824	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6825	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
6826	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6827	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6828	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6829	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6830	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
6831	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6832	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
6833	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6834	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
6835	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6836	#endif
6837	default:
6838	{
6839	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6840	AssertRCReturn(rc, rc);
6841
6842	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
6843	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6844	{
6845	Assert(pVM->hm.s.vmx.pRealModeTSS);
6846	Assert(PDMVmmDevHeapIsEnabled(pVM));
6847	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6848	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
6849	AssertRCReturn(rc, rc);
6850	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
6851	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
6852	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
6853	AssertRCReturn(rc, rc);
6854	}
6855	else
6856	{
6857	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
6858	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
6859	}
6860	break;
6861	}
6862	}
6863	break;
6864	}
6865
6866	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
6867	default:
6868	{
6869	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
6870	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
6871	break;
6872	}
6873	}
6874	return rc;
6875	}
6876
6877
6878	/**
6879	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
6880	*/
6881	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6882	{
6883	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6884
6885	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
6886	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
6887	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
6888	AssertRCReturn(rc, rc);
6889
6890	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and resume guest execution. */
6891	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6892	return VINF_SUCCESS;
6893	}
6894
6895
6896	/**
6897	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
6898	*/
6899	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6900	{
6901	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6902	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
6903	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6904	}
6905
6906
6907	/**
6908	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
6909	*/
6910	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6911	{
6912	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6913	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6914	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6915	AssertRCReturn(rc, rc);
6916
6917	pMixedCtx->rip += pVmxTransient->cbInstr;
6918	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6919
6920	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6921	return VINF_SUCCESS;
6922	}
6923
6924
6925	/**
6926	* VM-exit handler for WBINVD (VMX_EXIT_INVD). Unconditional VM-exit.
6927	*/
6928	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6929	{
6930	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6931	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6932	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6933	AssertRCReturn(rc, rc);
6934
6935	pMixedCtx->rip += pVmxTransient->cbInstr;
6936	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6937
6938	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6939	return VINF_SUCCESS;
6940	}
6941
6942
6943	/**
6944	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
6945	*/
6946	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6947	{
6948	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6949	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6950	if (RT_LIKELY(rc == VINF_SUCCESS))
6951	{
6952	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6953	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6954	AssertRCReturn(rc, rc);
6955	Assert(pVmxTransient->cbInstr == 2);
6956
6957	Log(("hmR0VmxExitCpuid: RIP=%#RX64\n", pMixedCtx->rip));
6958	pMixedCtx->rip += pVmxTransient->cbInstr;
6959	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6960	}
6961	else
6962	{
6963	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6964	rc = VERR_EM_INTERPRETER;
6965	}
6966	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6967	return rc;
6968	}
6969
6970
6971	/**
6972	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
6973	*/
6974	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6975	{
6976	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6977	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
6978	AssertRCReturn(rc, rc);
6979
6980	if (pMixedCtx->cr4 & X86_CR4_SMXE)
6981	return VINF_EM_RAW_EMULATE_INSTR;
6982
6983	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
6984	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6985	}
6986
6987
6988	/**
6989	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
6990	*/
6991	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6992	{
6993	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6994	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6995	AssertRCReturn(rc, rc);
6996
6997	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6998	if (RT_LIKELY(rc == VINF_SUCCESS))
6999	{
7000	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7001	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7002	AssertRCReturn(rc, rc);
7003	Assert(pVmxTransient->cbInstr == 2);
7004
7005	pMixedCtx->rip += pVmxTransient->cbInstr;
7006	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7007
7008	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7009	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7010	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7011	}
7012	else
7013	{
7014	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7015	rc = VERR_EM_INTERPRETER;
7016	}
7017	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7018	return rc;
7019	}
7020
7021
7022	/**
7023	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7024	*/
7025	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7026	{
7027	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7028	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7029	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7030	AssertRCReturn(rc, rc);
7031
7032	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7033	if (RT_LIKELY(rc == VINF_SUCCESS))
7034	{
7035	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7036	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7037	AssertRCReturn(rc, rc);
7038	Assert(pVmxTransient->cbInstr == 3);
7039
7040	pMixedCtx->rip += pVmxTransient->cbInstr;
7041	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7042
7043	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7044	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7045	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7046	}
7047	else
7048	{
7049	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7050	rc = VERR_EM_INTERPRETER;
7051	}
7052	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7053	return rc;
7054	}
7055
7056
7057	/**
7058	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7059	*/
7060	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7061	{
7062	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7063	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7064	rc \|= hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7065	AssertRCReturn(rc, rc);
7066
7067	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7068	if (RT_LIKELY(rc == VINF_SUCCESS))
7069	{
7070	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7071	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7072	AssertRCReturn(rc, rc);
7073	Assert(pVmxTransient->cbInstr == 2);
7074
7075	pMixedCtx->rip += pVmxTransient->cbInstr;
7076	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7077	}
7078	else
7079	{
7080	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7081	rc = VERR_EM_INTERPRETER;
7082	}
7083	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7084	return rc;
7085	}
7086
7087
7088	/**
7089	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7090	*/
7091	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7092	{
7093	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7094	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7095	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7096	AssertRCReturn(rc, rc);
7097
7098	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7099	rc = VBOXSTRICTRC_VAL(rc2);
7100	if (RT_LIKELY(rc == VINF_SUCCESS))
7101	{
7102	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7103	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7104	AssertRCReturn(rc, rc);
7105
7106	pMixedCtx->rip += pVmxTransient->cbInstr;
7107	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7108	}
7109	else
7110	{
7111	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %RGv failed with %Rrc\n",
7112	pVmxTransient->uExitQualification, rc));
7113	rc = VERR_EM_INTERPRETER;
7114	}
7115	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7116	return rc;
7117	}
7118
7119
7120	/**
7121	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7122	*/
7123	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7124	{
7125	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7126	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7127	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7128	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7129	AssertRCReturn(rc, rc);
7130
7131	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7132	if (RT_LIKELY(rc == VINF_SUCCESS))
7133	{
7134	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7135	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7136	AssertRCReturn(rc, rc);
7137
7138	pMixedCtx->rip += pVmxTransient->cbInstr;
7139	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7140	}
7141	else
7142	{
7143	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7144	rc = VERR_EM_INTERPRETER;
7145	}
7146	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7147	return rc;
7148	}
7149
7150
7151	/**
7152	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7153	*/
7154	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7155	{
7156	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7157	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7158	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7159	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7160	AssertRCReturn(rc, rc);
7161
7162	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7163	rc = VBOXSTRICTRC_VAL(rc2);
7164	if (RT_LIKELY( rc == VINF_SUCCESS
7165	\|\| rc == VINF_EM_HALT))
7166	{
7167	int rc3 = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7168	rc3 \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7169	AssertRCReturn(rc3, rc3);
7170
7171	pMixedCtx->rip += pVmxTransient->cbInstr;
7172	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7173
7174	if ( rc == VINF_EM_HALT
7175	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7176	{
7177	rc = VINF_SUCCESS;
7178	}
7179	}
7180	else
7181	{
7182	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7183	rc = VERR_EM_INTERPRETER;
7184	}
7185	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7186	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7187	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7188	return rc;
7189	}
7190
7191
7192	/**
7193	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7194	*/
7195	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7196	{
7197	/*
7198	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7199	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7200	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7201	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7202	*/
7203	AssertMsgFailed(("Unexpected RSM VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7204	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7205	}
7206
7207
7208	/**
7209	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7210	*/
7211	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7212	{
7213	/*
7214	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7215	* root operation. If we get there there is something funny going on.
7216	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7217	*/
7218	AssertMsgFailed(("Unexpected SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7219	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7220	}
7221
7222
7223	/**
7224	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7225	*/
7226	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7227	{
7228	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7229	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7230	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7231	}
7232
7233
7234	/**
7235	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7236	*/
7237	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7238	{
7239	/*
7240	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7241	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7242	* See Intel spec. 25.3 "Other Causes of VM-exits".
7243	*/
7244	AssertMsgFailed(("Unexpected SIPI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7245	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7246	}
7247
7248
7249	/**
7250	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7251	* VM-exit.
7252	*/
7253	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7254	{
7255	/*
7256	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7257	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7258	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7259	*/
7260	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7261	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7262	}
7263
7264
7265	/**
7266	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7267	* VM-exit.
7268	*/
7269	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7270	{
7271	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7272	return VINF_EM_RESET;
7273	}
7274
7275
7276	/**
7277	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7278	*/
7279	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7280	{
7281	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7282	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7283	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7284	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7285	AssertRCReturn(rc, rc);
7286
7287	pMixedCtx->rip++;
7288	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7289	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7290	rc = VINF_SUCCESS;
7291	else
7292	rc = VINF_EM_HALT;
7293
7294	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7295	return rc;
7296	}
7297
7298
7299	/**
7300	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7301	*/
7302	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7303	{
7304	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7305	return hmR0VmxInjectXcptUD(pVM, pVCpu, pMixedCtx);
7306	}
7307
7308
7309	/**
7310	* VM-exit handler for expiry of the VMX preemption timer.
7311	*/
7312	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7313	{
7314	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7315
7316	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7317	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7318	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7319
7320	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7321	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7322	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7323	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7324	}
7325
7326
7327	/**
7328	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7329	*/
7330	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7331	{
7332	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7333	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7334	/** @todo check if XSETBV is supported by the recompiler. */
7335	return VERR_EM_INTERPRETER;
7336	}
7337
7338
7339	/**
7340	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7341	*/
7342	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7343	{
7344	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7345	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7346	/** @todo implement EMInterpretInvpcid() */
7347	return VERR_EM_INTERPRETER;
7348	}
7349
7350
7351	/**
7352	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7353	* Error VM-exit.
7354	*/
7355	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7356	{
7357	uint32_t uIntrState;
7358	RTHCUINTREG uHCReg;
7359	uint64_t u64Val;
7360
7361	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7362	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7363	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7364	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7365	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7366	AssertRCReturn(rc, rc);
7367
7368	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7369	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7370	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7371	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7372
7373	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7374	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7375	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7376	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7377	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7378	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7379	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7380	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7381	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7382	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7383	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7384	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7385
7386	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7387
7388	return VERR_VMX_INVALID_GUEST_STATE;
7389	}
7390
7391
7392	/**
7393	* VM-exit handler for VM-entry failure due to an MSR-load
7394	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7395	*/
7396	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7397	{
7398	AssertMsgFailed(("Unexpected MSR-load exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7399	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7400	}
7401
7402
7403	/**
7404	* VM-exit handler for VM-entry failure due to a machine-check event
7405	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7406	*/
7407	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7408	{
7409	AssertMsgFailed(("Unexpected machine-check event exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7410	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7411	}
7412
7413
7414	/**
7415	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7416	* theory.
7417	*/
7418	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7419	{
7420	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason,
7421	pVM, pVCpu, pMixedCtx));
7422	return VERR_VMX_UNDEFINED_EXIT_CODE;
7423	}
7424
7425
7426	/**
7427	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7428	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7429	* Conditional VM-exit.
7430	*/
7431	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7432	{
7433	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7434	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7435	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7436	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7437	return VERR_EM_INTERPRETER;
7438	AssertMsgFailed(("Unexpected XDTR access. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7439	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7440	}
7441
7442
7443	/**
7444	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7445	*/
7446	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7447	{
7448	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7449	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7450	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7451	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7452	return VERR_EM_INTERPRETER;
7453	AssertMsgFailed(("Unexpected RDRAND exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7454	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7455	}
7456
7457
7458	/**
7459	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7460	*/
7461	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7462	{
7463	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7464	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7465	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7466	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7467	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7468	AssertRCReturn(rc, rc);
7469
7470	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7471	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7472	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7473	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7474
7475	/* Update RIP and continue guest execution. */
7476	if (RT_LIKELY(rc == VINF_SUCCESS))
7477	{
7478	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7479	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7480	AssertRCReturn(rc, rc);
7481
7482	Assert(pVmxTransient->cbInstr == 2);
7483	pMixedCtx->rip += pVmxTransient->cbInstr;
7484	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7485	}
7486	return rc;
7487	}
7488
7489
7490	/**
7491	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7492	*/
7493	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7494	{
7495	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7496	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7497	AssertRCReturn(rc, rc);
7498	Assert(pVmxTransient->cbInstr == 2);
7499
7500	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7501	if ( pVM->hm.s.fTPRPatchingActive
7502	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7503	{
7504	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7505	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7506	{
7507	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7508	AssertRC(rc);
7509	}
7510
7511	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7512	AssertRCReturn(rc, rc);
7513	pMixedCtx->rip += pVmxTransient->cbInstr;
7514	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7515	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7516	return VINF_SUCCESS;
7517	}
7518
7519	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7520	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS))
7521	{
7522	switch (pMixedCtx->ecx)
7523	{
7524	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7525	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7526	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7527	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FS_BASE_MSR; break;
7528	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_GS_BASE_MSR; break;
7529	}
7530	}
7531	#ifdef DEBUG
7532	else
7533	{
7534	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7535	switch (pMixedCtx->ecx)
7536	{
7537	case MSR_IA32_SYSENTER_CS:
7538	case MSR_IA32_SYSENTER_EIP:
7539	case MSR_IA32_SYSENTER_ESP:
7540	case MSR_K8_FS_BASE:
7541	case MSR_K8_GS_BASE:
7542	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7543	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7544	case MSR_K8_LSTAR:
7545	case MSR_K6_STAR:
7546	case MSR_K8_SF_MASK:
7547	case MSR_K8_TSC_AUX:
7548	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7549	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7550	}
7551	}
7552	#endif
7553
7554	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7555	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7556	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7557	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7558	AssertRCReturn(rc, rc);
7559
7560	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7561	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7562	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7563
7564	/* Update guest-state and continue execution. */
7565	if (RT_LIKELY(rc == VINF_SUCCESS))
7566	{
7567	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7568	AssertRCReturn(rc, rc);
7569
7570	pMixedCtx->rip += pVmxTransient->cbInstr;
7571	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7572
7573	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7574	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7575	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7576	{
7577	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7578	}
7579	}
7580	return rc;
7581	}
7582
7583
7584	/**
7585	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7586	*/
7587	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7588	{
7589	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7590	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7591	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7592	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7593	return VERR_EM_INTERPRETER;
7594	AssertMsgFailed(("Unexpected PAUSE exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7595	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7596	}
7597
7598
7599	/**
7600	* VM-exit handler for when the TPR value is lowered below the specified
7601	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7602	*/
7603	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7604	{
7605	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7606	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7607
7608	/*
7609	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7610	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and
7611	* resume guest execution.
7612	*/
7613	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7614	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7615	return VINF_SUCCESS;
7616	}
7617
7618
7619	/**
7620	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7621	* VM-exit.
7622	*
7623	* @retval VINF_SUCCESS when guest execution can continue.
7624	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7625	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7626	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7627	* recompiler.
7628	*/
7629	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7630	{
7631	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7632	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7633	AssertRCReturn(rc, rc);
7634
7635	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7636	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7637	switch (uAccessType)
7638	{
7639	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7640	{
7641	#if 0
7642	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7643	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7644	#else
7645	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
7646	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7647	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7648	#endif
7649	AssertRCReturn(rc, rc);
7650
7651	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7652	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7653	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7654	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7655
7656	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7657	{
7658	case 0: /* CR0 */
7659	Log(("CR0 write rc=%d\n", rc));
7660	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7661	break;
7662	case 2: /* CR2 */
7663	Log(("CR2 write rc=%d\n", rc));
7664	break;
7665	case 3: /* CR3 */
7666	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7667	Log(("CR3 write rc=%d\n", rc));
7668	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7669	break;
7670	case 4: /* CR4 */
7671	Log(("CR4 write rc=%d\n", rc));
7672	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7673	break;
7674	case 8: /* CR8 */
7675	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7676	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7677	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7678	break;
7679	default:
7680	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7681	break;
7682	}
7683
7684	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7685	break;
7686	}
7687
7688	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7689	{
7690	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7691	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7692	AssertRCReturn(rc, rc);
7693	Assert( !pVM->hm.s.fNestedPaging
7694	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7695	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7696
7697	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7698	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7699	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7700
7701	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7702	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7703	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7704	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7705	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7706	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7707	break;
7708	}
7709
7710	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7711	{
7712	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7713	AssertRCReturn(rc, rc);
7714	rc = EMInterpretCLTS(pVM, pVCpu);
7715	AssertRCReturn(rc, rc);
7716	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7717	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7718	Log(("CRX CLTS write rc=%d\n", rc));
7719	break;
7720	}
7721
7722	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7723	{
7724	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7725	AssertRCReturn(rc, rc);
7726	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7727	if (RT_LIKELY(rc == VINF_SUCCESS))
7728	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7729	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7730	Log(("CRX LMSW write rc=%d\n", rc));
7731	break;
7732	}
7733
7734	default:
7735	{
7736	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7737	return VERR_VMX_UNEXPECTED_EXCEPTION;
7738	}
7739	}
7740
7741	/* Validate possible error codes. */
7742	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3);
7743	if (RT_SUCCESS(rc))
7744	{
7745	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7746	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7747	AssertRCReturn(rc2, rc2);
7748	pMixedCtx->rip += pVmxTransient->cbInstr;
7749	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7750	}
7751
7752	return rc;
7753	}
7754
7755
7756	/**
7757	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7758	* VM-exit.
7759	*/
7760	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7761	{
7762	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7763
7764	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7765	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7766	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7767	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7768	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7769	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7770	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7771	AssertRCReturn(rc, rc);
7772
7773	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7774	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7775	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7776	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7777	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7778	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7779	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7780
7781	/* I/O operation lookup arrays. */
7782	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7783	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7784
7785	const uint32_t cbSize = s_aIOSize[uIOWidth];
7786	const uint32_t cbInstr = pVmxTransient->cbInstr;
7787	if (fIOString)
7788	{
7789	/* INS/OUTS - I/O String instruction. */
7790	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7791	/** @todo for now manually disassemble later optimize by getting the fields from
7792	* the VMCS. */
7793	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7794	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7795	* segment prefix info. */
7796	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7797	if (RT_SUCCESS(rc))
7798	{
7799	if (fIOWrite)
7800	{
7801	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7802	(DISCPUMODE)pDis->uAddrMode, cbSize);
7803	rc = VBOXSTRICTRC_VAL(rc2);
7804	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7805	}
7806	else
7807	{
7808	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7809	(DISCPUMODE)pDis->uAddrMode, cbSize);
7810	rc = VBOXSTRICTRC_VAL(rc2);
7811	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7812	}
7813	}
7814	else
7815	{
7816	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7817	rc = VINF_EM_RAW_EMULATE_INSTR;
7818	}
7819	}
7820	else
7821	{
7822	/* IN/OUT - I/O instruction. */
7823	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7824	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7825	if (fIOWrite)
7826	{
7827	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7828	rc = VBOXSTRICTRC_VAL(rc2);
7829	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7830	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7831	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7832	}
7833	else
7834	{
7835	uint32_t u32Result = 0;
7836	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7837	rc = VBOXSTRICTRC_VAL(rc2);
7838	if (IOM_SUCCESS(rc))
7839	{
7840	/* Save result of I/O IN instr. in AL/AX/EAX. */
7841	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7842	}
7843	else if (rc == VINF_IOM_R3_IOPORT_READ)
7844	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7845	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7846	}
7847	}
7848
7849	if (IOM_SUCCESS(rc))
7850	{
7851	pMixedCtx->rip += cbInstr;
7852	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7853	if (RT_LIKELY(rc == VINF_SUCCESS))
7854	{
7855	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx); /* For DR7. */
7856	AssertRCReturn(rc, rc);
7857
7858	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7859	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7860	{
7861	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7862	for (unsigned i = 0; i < 4; i++)
7863	{
7864	unsigned uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7865	if ( ( uIOPort >= pMixedCtx->dr[i]
7866	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7867	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7868	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7869	{
7870	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7871	uint64_t uDR6 = ASMGetDR6();
7872
7873	/* Clear all breakpoint status flags and set the one we just hit. */
7874	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
7875	uDR6 \|= (uint64_t)RT_BIT(i);
7876
7877	/*
7878	* Note: AMD64 Architecture Programmer's Manual 13.1:
7879	* Bits 15:13 of the DR6 register is never cleared by the processor and must
7880	* be cleared by software after the contents have been read.
7881	*/
7882	ASMSetDR6(uDR6);
7883
7884	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
7885	pMixedCtx->dr[7] &= ~X86_DR7_GD;
7886
7887	/* Paranoia. */
7888	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
7889	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
7890	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
7891
7892	/* Resync DR7 */
7893	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
7894	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7895
7896	/* Inject #DB and get on with guest execution. */
7897	rc = hmR0VmxInjectXcptDB(pVM, pVCpu, pMixedCtx);
7898	AssertRCReturn(rc, rc);
7899	break;
7900	}
7901	}
7902	}
7903	}
7904	}
7905
7906	#ifdef DEBUG
7907	if (rc == VINF_IOM_R3_IOPORT_READ)
7908	Assert(!fIOWrite);
7909	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
7910	Assert(fIOWrite);
7911	else
7912	{
7913	AssertMsg( RT_FAILURE(rc)
7914	\|\| rc == VINF_SUCCESS
7915	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
7916	\|\| rc == VINF_EM_RAW_GUEST_TRAP
7917	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
7918	}
7919	#endif
7920
7921	return rc;
7922	}
7923
7924
7925	/**
7926	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
7927	* VM-exit.
7928	*/
7929	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7930	{
7931	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7932
7933	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
7934	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7935	AssertRCReturn(rc, rc);
7936	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
7937	{
7938	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
7939	AssertRCReturn(rc, rc);
7940	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
7941	{
7942	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
7943	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7944	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7945	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7946	{
7947	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
7948	pVCpu->hm.s.Event.fPending = true;
7949	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
7950	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
7951	AssertRCReturn(rc, rc);
7952	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringErrorCode))
7953	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
7954	else
7955	pVCpu->hm.s.Event.u32ErrCode = 0;
7956	}
7957	}
7958	}
7959	/** @todo Emulate task switch someday, currently just going back to ring-3 for
7960	* emulation. */
7961	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7962	return VERR_EM_INTERPRETER;
7963	}
7964
7965
7966	/**
7967	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
7968	*/
7969	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7970	{
7971	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7972	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
7973	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
7974	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7975	AssertRCReturn(rc, rc);
7976	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
7977	return VINF_EM_DBG_STOP;
7978	}
7979
7980
7981	/**
7982	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
7983	*/
7984	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7985	{
7986	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7987	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7988
7989	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7990	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
7991	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7992	return VINF_SUCCESS;
7993	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7994	return rc;
7995
7996	#if 0
7997	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
7998	* just sync the whole thing. */
7999	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8000	#else
8001	/* Aggressive state sync. for now. */
8002	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8003	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8004	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8005	#endif
8006	AssertRCReturn(rc, rc);
8007
8008	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8009	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8010	switch (uAccessType)
8011	{
8012	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8013	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8014	{
8015	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8016	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8017	{
8018	AssertMsgFailed(("hmR0VmxExitApicAccess: can't touch TPR offset while using TPR shadowing.\n"));
8019	}
8020
8021	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8022	GCPhys &= PAGE_BASE_GC_MASK;
8023	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8024	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8025	CPUMCTX2CORE(pMixedCtx), GCPhys);
8026	rc = VBOXSTRICTRC_VAL(rc2);
8027	Log(("ApicAccess %RGp %#x\n", GCPhys, VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8028	if ( rc == VINF_SUCCESS
8029	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8030	\|\| rc == VERR_PAGE_NOT_PRESENT)
8031	{
8032	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8033	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8034	rc = VINF_SUCCESS;
8035	}
8036	break;
8037	}
8038
8039	default:
8040	rc = VINF_EM_RAW_EMULATE_INSTR;
8041	break;
8042	}
8043
8044	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8045	return rc;
8046	}
8047
8048
8049	/**
8050	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8051	* VM-exit.
8052	*/
8053	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8054	{
8055	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8056
8057	/* We should -not- get this VM-exit if the guest is debugging. */
8058	if (CPUMIsGuestDebugStateActive(pVCpu))
8059	{
8060	AssertMsgFailed(("Unexpected MOV DRx exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
8061	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8062	}
8063
8064	int rc = VERR_INTERNAL_ERROR_5;
8065	if ( !DBGFIsStepping(pVCpu)
8066	&& !CPUMIsHyperDebugStateActive(pVCpu))
8067	{
8068	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
8069
8070	/* Don't intercept MOV DRx. */
8071	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8072	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8073	AssertRCReturn(rc, rc);
8074
8075	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8076	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8077	AssertRC(rc);
8078	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8079
8080	#ifdef VBOX_WITH_STATISTICS
8081	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8082	AssertRCReturn(rc, rc);
8083	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8084	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8085	else
8086	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8087	#endif
8088	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8089	return VINF_SUCCESS;
8090	}
8091
8092	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
8093	* time and restore DRx registers afterwards */
8094	/*
8095	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8096	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8097	*/
8098	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8099	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8100	AssertRCReturn(rc, rc);
8101
8102	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8103	{
8104	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8105	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8106	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8107	if (RT_SUCCESS(rc))
8108	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8109	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8110	}
8111	else
8112	{
8113	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8114	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8115	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8116	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8117	}
8118
8119	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8120	if (RT_SUCCESS(rc))
8121	{
8122	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8123	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8124	AssertRCReturn(rc2, rc2);
8125	pMixedCtx->rip += pVmxTransient->cbInstr;
8126	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8127	}
8128	return rc;
8129	}
8130
8131
8132	/**
8133	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8134	* Conditional VM-exit.
8135	*/
8136	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8137	{
8138	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8139	Assert(pVM->hm.s.fNestedPaging);
8140
8141	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8142	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8143	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8144	return VINF_SUCCESS;
8145	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8146	return rc;
8147
8148	RTGCPHYS GCPhys = 0;
8149	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8150
8151	#if 0
8152	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
8153	#else
8154	/* Aggressive state sync. for now. */
8155	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8156	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8157	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8158	#endif
8159	AssertRCReturn(rc, rc);
8160
8161	/*
8162	* If we succeed, resume guest execution.
8163	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8164	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8165	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8166	* weird case. See @bugref{6043}.
8167	*/
8168	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8169	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8170	rc = VBOXSTRICTRC_VAL(rc2);
8171	if ( rc == VINF_SUCCESS
8172	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8173	\|\| rc == VERR_PAGE_NOT_PRESENT)
8174	{
8175	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8176	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8177	return VINF_SUCCESS;
8178	}
8179	return rc;
8180	}
8181
8182
8183	/**
8184	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8185	* VM-exit.
8186	*/
8187	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8188	{
8189	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8190	Assert(pVM->hm.s.fNestedPaging);
8191
8192	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8193	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8194	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8195	return VINF_SUCCESS;
8196	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8197	return rc;
8198
8199	RTGCPHYS GCPhys = 0;
8200	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8201	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8202	#if 0
8203	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
8204	#else
8205	/* Aggressive state sync. for now. */
8206	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8207	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8208	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8209	#endif
8210	AssertRCReturn(rc, rc);
8211
8212	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8213	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8214
8215	RTGCUINT uErrorCode = 0;
8216	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8217	uErrorCode \|= X86_TRAP_PF_ID;
8218	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8219	uErrorCode \|= X86_TRAP_PF_RW;
8220	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8221	uErrorCode \|= X86_TRAP_PF_P;
8222
8223	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8224	TRPMSetErrorCode(pVCpu, uErrorCode);
8225	TRPMSetFaultAddress(pVCpu, GCPhys);
8226
8227	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8228	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8229
8230	/* Handle the pagefault trap for the nested shadow table. */
8231	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8232	TRPMResetTrap(pVCpu);
8233
8234	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8235	if ( rc == VINF_SUCCESS
8236	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8237	\|\| rc == VERR_PAGE_NOT_PRESENT)
8238	{
8239	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8240	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8241	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8242	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8243	return VINF_SUCCESS;
8244	}
8245
8246	Log(("EPT return to ring-3 rc=%d\n"));
8247	return rc;
8248	}
8249
8250
8251	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8252	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8253	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8254	/**
8255	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8256	*/
8257	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8258	{
8259	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8260
8261	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
8262	AssertRCReturn(rc, rc);
8263	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8264
8265	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8266	{
8267	/* Old-style FPU error reporting needs some extra work. */
8268	/** @todo don't fall back to the recompiler, but do it manually. */
8269	return VERR_EM_INTERPRETER;
8270	}
8271	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8272	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8273	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8274	AssertRCReturn(rc, rc);
8275	return rc;
8276	}
8277
8278
8279	/**
8280	* VM-exit exception handler for #BP (Breakpoint exception).
8281	*/
8282	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8283	{
8284	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8285
8286	/** @todo Try optimize this by not saving the entire guest state unless
8287	* really needed. */
8288	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8289	AssertRCReturn(rc, rc);
8290	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8291
8292	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8293	if (rc == VINF_EM_RAW_GUEST_TRAP)
8294	{
8295	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8296	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8297	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8298	AssertRCReturn(rc, rc);
8299
8300	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8301	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8302	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8303	AssertRCReturn(rc, rc);
8304	}
8305
8306	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8307	return rc;
8308	}
8309
8310
8311	/**
8312	* VM-exit exception handler for #DB (Debug exception).
8313	*/
8314	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8315	{
8316	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8317
8318	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8319	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8320	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8321	AssertRCReturn(rc, rc);
8322
8323	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8324	uint64_t uDR6 = X86_DR6_INIT_VAL;
8325	uDR6 \|= (pVmxTransient->uExitQualification
8326	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8327	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8328	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8329	if (rc == VINF_EM_RAW_GUEST_TRAP)
8330	{
8331	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8332	pMixedCtx->dr[6] = uDR6;
8333
8334	if (CPUMIsGuestDebugStateActive(pVCpu))
8335	ASMSetDR6(pMixedCtx->dr[6]);
8336
8337	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8338	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8339
8340	/* Paranoia. */
8341	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8342	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8343	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8344
8345	/* Resync DR7. */
8346	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8347
8348	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8349	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8350	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8351	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8352	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8353	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8354	AssertRCReturn(rc,rc);
8355	return rc;
8356	}
8357	/* Return to ring 3 to deal with the debug exit code. */
8358	return rc;
8359	}
8360
8361
8362	/**
8363	* VM-exit exception handler for #NM (Device-not-available exception: floating
8364	* point exception).
8365	*/
8366	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8367	{
8368	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8369
8370	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8371	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8372	#endif
8373
8374	/* We require CR0 and EFER. EFER is always up-to-date. */
8375	int rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8376	AssertRCReturn(rc, rc);
8377
8378	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8379	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8380	if (rc == VINF_SUCCESS)
8381	{
8382	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8383	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8384	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8385	return VINF_SUCCESS;
8386	}
8387
8388	/* Forward #NM to the guest. */
8389	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8390	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8391	AssertRCReturn(rc, rc);
8392	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8393	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8394	pVmxTransient->cbInstr, 0 /* error code */);
8395	AssertRCReturn(rc, rc);
8396	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8397	return rc;
8398	}
8399
8400
8401	/**
8402	* VM-exit exception handler for #GP (General-protection exception).
8403	*
8404	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8405	*/
8406	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8407	{
8408	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8409
8410	int rc = VERR_INTERNAL_ERROR_5;
8411
8412	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8413	{
8414	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8415	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8416	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8417	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8418	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8419	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8420	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8421	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8422	AssertRCReturn(rc, rc);
8423	return rc;
8424	#else
8425	/* We don't intercept #GP. */
8426	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8427	return VERR_VMX_UNEXPECTED_EXCEPTION;
8428	#endif
8429	}
8430
8431	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8432	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
8433
8434	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8435	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8436	AssertRCReturn(rc, rc);
8437
8438	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8439	unsigned int cbOp = 0;
8440	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8441	if (RT_SUCCESS(rc))
8442	{
8443	rc = VINF_SUCCESS;
8444	Assert(cbOp == pDis->cbInstr);
8445	Log2(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8446	switch (pDis->pCurInstr->uOpcode)
8447	{
8448	case OP_CLI:
8449	pMixedCtx->eflags.Bits.u1IF = 0;
8450	pMixedCtx->rip += pDis->cbInstr;
8451	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8452	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8453	break;
8454
8455	case OP_STI:
8456	pMixedCtx->eflags.Bits.u1IF = 1;
8457	pMixedCtx->rip += pDis->cbInstr;
8458	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8459	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8460	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8461	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8462	break;
8463
8464	case OP_HLT:
8465	rc = VINF_EM_HALT;
8466	pMixedCtx->rip += pDis->cbInstr;
8467	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8468	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8469	break;
8470
8471	case OP_POPF:
8472	{
8473	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8474	uint32_t cbParm = 0;
8475	uint32_t uMask = 0;
8476	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8477	{
8478	cbParm = 4;
8479	uMask = 0xffffffff;
8480	}
8481	else
8482	{
8483	cbParm = 2;
8484	uMask = 0xffff;
8485	}
8486
8487	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8488	RTGCPTR GCPtrStack = 0;
8489	X86EFLAGS uEflags;
8490	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8491	&GCPtrStack);
8492	if (RT_SUCCESS(rc))
8493	{
8494	Assert(sizeof(uEflags.u32) >= cbParm);
8495	uEflags.u32 = 0;
8496	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8497	}
8498	if (RT_FAILURE(rc))
8499	{
8500	rc = VERR_EM_INTERPRETER;
8501	break;
8502	}
8503	Log(("POPF %x -> %RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8504	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8505	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8506	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8507	pMixedCtx->eflags.Bits.u1RF = 0;
8508	pMixedCtx->esp += cbParm;
8509	pMixedCtx->esp &= uMask;
8510	pMixedCtx->rip += pDis->cbInstr;
8511	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8512	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8513	break;
8514	}
8515
8516	case OP_PUSHF:
8517	{
8518	uint32_t cbParm = 0;
8519	uint32_t uMask = 0;
8520	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8521	{
8522	cbParm = 4;
8523	uMask = 0xffffffff;
8524	}
8525	else
8526	{
8527	cbParm = 2;
8528	uMask = 0xffff;
8529	}
8530
8531	/* Get the stack pointer & push the contents of eflags onto the stack. */
8532	RTGCPTR GCPtrStack = 0;
8533	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8534	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8535	if (RT_FAILURE(rc))
8536	{
8537	rc = VERR_EM_INTERPRETER;
8538	break;
8539	}
8540	X86EFLAGS uEflags;
8541	uEflags = pMixedCtx->eflags;
8542	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8543	uEflags.Bits.u1RF = 0;
8544	uEflags.Bits.u1VM = 0;
8545
8546	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8547	if (RT_FAILURE(rc))
8548	{
8549	rc = VERR_EM_INTERPRETER;
8550	break;
8551	}
8552	Log(("PUSHF %x -> %RGv\n", uEflags.u, GCPtrStack));
8553	pMixedCtx->esp -= cbParm;
8554	pMixedCtx->esp &= uMask;
8555	pMixedCtx->rip += pDis->cbInstr;
8556	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8557	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8558	break;
8559	}
8560
8561	case OP_IRET:
8562	{
8563	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8564	* instruction reference. */
8565	RTGCPTR GCPtrStack = 0;
8566	uint32_t uMask = 0xffff;
8567	uint16_t aIretFrame[3];
8568	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8569	{
8570	rc = VERR_EM_INTERPRETER;
8571	break;
8572	}
8573	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8574	&GCPtrStack);
8575	if (RT_SUCCESS(rc))
8576	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8577	if (RT_FAILURE(rc))
8578	{
8579	rc = VERR_EM_INTERPRETER;
8580	break;
8581	}
8582	pMixedCtx->eip = 0;
8583	pMixedCtx->ip = aIretFrame[0];
8584	pMixedCtx->cs.Sel = aIretFrame[1];
8585	pMixedCtx->cs.ValidSel = aIretFrame[1];
8586	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8587	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8588	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8589	pMixedCtx->sp += sizeof(aIretFrame);
8590	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8591	\| HM_CHANGED_GUEST_RFLAGS;
8592	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8593	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8594	break;
8595	}
8596
8597	case OP_INT:
8598	{
8599	uint16_t uVector = pDis->Param1.uValue & 0xff;
8600	rc = hmR0VmxInjectIntN(pVM, pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8601	AssertRCReturn(rc, rc);
8602	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8603	break;
8604	}
8605
8606	case OP_INTO:
8607	{
8608	if (pMixedCtx->eflags.Bits.u1OF)
8609	{
8610	rc = hmR0VmxInjectXcptOF(pVM, pVCpu, pMixedCtx, pDis->cbInstr);
8611	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8612	}
8613	break;
8614	}
8615
8616	default:
8617	{
8618	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8619	EMCODETYPE_SUPERVISOR);
8620	rc = VBOXSTRICTRC_VAL(rc2);
8621	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8622	Log2(("#GP rc=%Rrc\n", rc));
8623	break;
8624	}
8625	}
8626	}
8627	else
8628	rc = VERR_EM_INTERPRETER;
8629
8630	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT
8631	\|\| rc == VINF_EM_RESET /* injection caused triple fault */,
8632	("#GP Unexpected rc=%Rrc\n", rc));
8633	return rc;
8634	}
8635
8636
8637	/**
8638	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8639	* the exception reported in the VMX transient structure back into the VM.
8640	*
8641	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8642	* VMX transient structure to be up-to-date.
8643	*/
8644	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8645	{
8646	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8647
8648	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8649	hmR0VmxCheckExitDueToEventDelivery(). */
8650	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8651	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8652	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8653	AssertRCReturn(rc, rc);
8654	return rc;
8655	}
8656
8657
8658	/**
8659	* VM-exit exception handler for #PF (Page-fault exception).
8660	*/
8661	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8662	{
8663	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8664
8665	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8666	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8667	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8668	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8669	AssertRCReturn(rc, rc);
8670
8671	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8672	if (pVM->hm.s.fNestedPaging)
8673	{
8674	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8675	{
8676	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8677	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8678	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8679	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8680	AssertRCReturn(rc, rc);
8681	}
8682	else
8683	{
8684	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8685	Assert(!pVCpu->hm.s.Event.fPending);
8686	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8687	}
8688	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8689	return rc;
8690	}
8691	#else
8692	Assert(!pVM->hm.s.fNestedPaging);
8693	#endif
8694
8695	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8696	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8697	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8698	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8699	AssertRCReturn(rc, rc);
8700	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8701	if ( pVM->hm.s.fTRPPatchingAllowed
8702	&& pVM->hm.s.pGuestPatchMem
8703	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8704	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8705	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8706	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8707	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8708	{
8709	RTGCPHYS GCPhys;
8710	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8711	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8712	if ( rc == VINF_SUCCESS
8713	&& GCPhys == GCPhysApicBase)
8714	{
8715	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8716	AssertRCReturn(rc, rc);
8717
8718	/* Only attempt to patch the instruction once. */
8719	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8720	if (!pPatch)
8721	return VINF_EM_HM_PATCH_TPR_INSTR;
8722	}
8723	}
8724	#endif
8725
8726	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8727	TRPMSetFaultAddress(pVCpu, pVmxTransient->uExitQualification);
8728	TRPMSetErrorCode(pVCpu, pVmxTransient->uExitIntrErrorCode);
8729
8730	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8731	AssertRCReturn(rc, rc);
8732
8733	/* Forward it to the trap handler first. */
8734	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8735	(RTGCPTR)pVmxTransient->uExitQualification);
8736
8737	Log(("#PF: cr2=%RGv cs:rip=%04x:%RGv errorcode %#RX32 rc=%d\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8738	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, rc));
8739
8740	if (rc == VINF_SUCCESS)
8741	{
8742	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8743	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8744	* memory? We don't update the whole state here... */
8745	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8746	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8747
8748	TRPMResetTrap(pVCpu);
8749	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8750	return rc;
8751	}
8752	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8753	{
8754	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8755	{
8756	/* It's a guest page fault and needs to be reflected to the guest. */
8757	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8758	TRPMResetTrap(pVCpu);
8759	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8760	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8761	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8762	pVmxTransient->cbInstr, uGstErrorCode);
8763	AssertRCReturn(rc, rc);
8764	}
8765	else
8766	{
8767	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8768	Assert(!pVCpu->hm.s.Event.fPending);
8769	TRPMResetTrap(pVCpu);
8770	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8771	}
8772	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8773	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8774	return rc;
8775	}
8776
8777	TRPMResetTrap(pVCpu);
8778	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8779	return rc;
8780	}
8781

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45467

Download in other formats: