HMVMXR0.cpp@ 45478

Last change on this file since 45478 was 45478, checked in by vboxsync, 12 years ago
VMM/VMMR0: more consistent names.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 369.4 KB

Line
1	/* $Id: HMVMXR0.cpp 45478 2013-04-11 09:20:01Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
39	#endif
40
41	/*******************************************************************************
42	* Defined Constants And Macros *
43	*******************************************************************************/
44	#if defined(RT_ARCH_AMD64)
45	# define VMX_IS_64BIT_HOST_MODE() (true)
46	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
47	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
48	#else
49	# define VMX_IS_64BIT_HOST_MODE() (false)
50	#endif
51
52	#define VMX_SEL_UNUSABLE RT_BIT(16)
53
54	/**
55	* Updated-guest-state flags.
56	*/
57	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
58	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
59	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
60	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
61	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
62	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
63	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
64	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
65	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
66	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
67	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
68	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
69	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
70	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
71	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
72	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
73	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
74	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
75	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(18)
76	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(19)
77	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(20)
78	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
79	\| VMX_UPDATED_GUEST_RIP \
80	\| VMX_UPDATED_GUEST_RSP \
81	\| VMX_UPDATED_GUEST_RFLAGS \
82	\| VMX_UPDATED_GUEST_CR0 \
83	\| VMX_UPDATED_GUEST_CR3 \
84	\| VMX_UPDATED_GUEST_CR4 \
85	\| VMX_UPDATED_GUEST_GDTR \
86	\| VMX_UPDATED_GUEST_IDTR \
87	\| VMX_UPDATED_GUEST_LDTR \
88	\| VMX_UPDATED_GUEST_TR \
89	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
90	\| VMX_UPDATED_GUEST_DEBUG \
91	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
92	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
93	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
94	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
95	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
96	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
97	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
98	\| VMX_UPDATED_GUEST_APIC_STATE)
99
100	/**
101	* Flags to skip redundant reads of some common VMCS fields.
102	*/
103	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
104	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
105	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
106	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
107	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
108	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
109
110	/*
111	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
112	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
113	* we have Nested Paging support.
114	*/
115	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
116	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
117	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
118	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
119	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
120	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
121	\| RT_BIT(X86_XCPT_XF))
122
123	/* Maximum VM-instruction error number. */
124	#define VMX_INSTR_ERROR_MAX 28
125
126	/*******************************************************************************
127	* Structures and Typedefs *
128	*******************************************************************************/
129	/**
130	* A state structure for holding miscellaneous information across
131	* VMX non-root operation and restored after the transition.
132	*/
133	typedef struct VMXTRANSIENT
134	{
135	/** The host's rflags/eflags. */
136	RTCCUINTREG uEFlags;
137	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
138	uint64_t u64LStarMsr;
139	/** The guest's TPR value used for TPR shadowing. */
140	uint8_t u8GuestTpr;
141
142	/** The basic VM-exit reason. */
143	uint16_t uExitReason;
144	/** The VM-exit exit qualification. */
145	RTGCUINTPTR uExitQualification;
146	/** The VM-exit interruption error code. */
147	uint32_t uExitIntrErrorCode;
148
149	/** The VM-exit interruption-information field. */
150	uint32_t uExitIntrInfo;
151	/** Whether the VM-entry failed or not. */
152	bool fVMEntryFailed;
153	/** The VM-exit instruction-length field. */
154	uint32_t cbInstr;
155
156	/** The VM-entry interruption-information field. */
157	uint32_t uEntryIntrInfo;
158	/** The VM-entry exception error code field. */
159	uint32_t uEntryXcptErrorCode;
160	/** The VM-entry instruction length field. */
161	uint32_t cbEntryInstr;
162
163	/** IDT-vectoring information field. */
164	uint32_t uIdtVectoringInfo;
165	/** IDT-vectoring error code. */
166	uint32_t uIdtVectoringErrorCode;
167
168	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
169	uint32_t fVmcsFieldsRead;
170	/** Whether TSC-offsetting should be setup before VM-entry. */
171	bool fUpdateTscOffsettingAndPreemptTimer;
172	/** Whether the VM-exit was caused by a page-fault during delivery of a
173	* contributary exception or a page-fault. */
174	bool fVectoringPF;
175	} VMXTRANSIENT, *PVMXTRANSIENT;
176
177	/**
178	* MSR-bitmap read permissions.
179	*/
180	typedef enum VMXMSREXITREAD
181	{
182	/** Reading this MSR causes a VM-exit. */
183	VMXMSREXIT_INTERCEPT_READ = 0xb,
184	/** Reading this MSR does not cause a VM-exit. */
185	VMXMSREXIT_PASSTHRU_READ
186	} VMXMSREXITREAD;
187
188	/**
189	* MSR-bitmap write permissions.
190	*/
191	typedef enum VMXMSREXITWRITE
192	{
193	/** Writing to this MSR causes a VM-exit. */
194	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
195	/** Writing to this MSR does not cause a VM-exit. */
196	VMXMSREXIT_PASSTHRU_WRITE
197	} VMXMSREXITWRITE;
198
199	/*******************************************************************************
200	* Internal Functions *
201	*******************************************************************************/
202	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
203	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
204	uint32_t cbInstr, uint32_t u32ErrCode);
205	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
206	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
207	#endif
208	#if 0
209	DECLINLINE(int) hmR0VmxHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
210	unsigned rcReason);
211	#endif
212
213	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
214	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
215	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
216	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
217	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
218	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
219	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
220	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
221	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
222	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
223	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
224	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264
265	/*******************************************************************************
266	* Global Variables *
267	*******************************************************************************/
268	/** @todo Move this to hm_vmx.h. */
269	/**
270	* VM-exit handler.
271	*
272	* @returns VBox status code.
273	* @param pVM Pointer to the VM.
274	* @param pVCpu Pointer to the VMCPU.
275	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
276	* out-of-sync. Make sure to update the required
277	* fields before using them.
278	* @param pVmxTransient Pointer to the VMX-transient structure.
279	*/
280	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	/** Pointer to VM-exit handler. */
282	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
283
284	static const PFNVMEXITHANDLER s_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
285	{
286	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
287	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
288	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
289	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
290	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
291	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
292	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
293	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
294	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
295	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
296	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
297	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
298	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
299	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
300	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
301	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
302	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
303	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
304	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitInjectXcptUD,
305	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitInjectXcptUD,
306	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitInjectXcptUD,
307	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitInjectXcptUD,
308	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitInjectXcptUD,
309	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitInjectXcptUD,
310	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitInjectXcptUD,
311	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitInjectXcptUD,
312	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitInjectXcptUD,
313	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitInjectXcptUD,
314	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
315	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
316	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
317	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
318	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
319	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
320	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
321	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
322	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
323	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
324	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
325	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
326	/* 40 UNDEFINED */ hmR0VmxExitPause,
327	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
328	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
329	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
330	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
331	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
332	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
333	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
334	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
335	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
336	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitInjectXcptUD,
337	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
338	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
339	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInjectXcptUD,
340	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
341	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
342	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
343	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
344	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
345	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitInjectXcptUD
346	};
347
348	static const char* const s_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
349	{
350	/* 0 */ "(Not Used)",
351	/* 1 */ "VMCALL executed in VMX root operation.",
352	/* 2 */ "VMCLEAR with invalid physical address.",
353	/* 3 */ "VMCLEAR with VMXON pointer.",
354	/* 4 */ "VMLAUNCH with non-clear VMCS.",
355	/* 5 */ "VMRESUME with non-launched VMCS.",
356	/* 6 */ "VMRESUME after VMXOFF",
357	/* 7 */ "VM entry with invalid control fields.",
358	/* 8 */ "VM entry with invalid host state fields.",
359	/* 9 */ "VMPTRLD with invalid physical address.",
360	/* 10 */ "VMPTRLD with VMXON pointer.",
361	/* 11 */ "VMPTRLD with incorrect revision identifier.",
362	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
363	/* 13 */ "VMWRITE to read-only VMCS component.",
364	/* 14 */ "(Not Used)",
365	/* 15 */ "VMXON executed in VMX root operation.",
366	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
367	/* 17 */ "VM entry with non-launched executing VMCS.",
368	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
369	/* 19 */ "VMCALL with non-clear VMCS.",
370	/* 20 */ "VMCALL with invalid VM-exit control fields.",
371	/* 21 */ "(Not Used)",
372	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
373	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
374	/* 24 */ "VMCALL with invalid SMM-monitor features.",
375	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
376	/* 26 */ "VM entry with events blocked by MOV SS.",
377	/* 27 */ "(Not Used)",
378	/* 28 */ "Invalid operand to INVEPT/INVVPID."
379	};
380
381
382	/**
383	* Updates the VM's last error record. If there was a VMX instruction error,
384	* reads the error data from the VMCS and updates VCPU's last error record as
385	* well.
386	*
387	* @param pVM Pointer to the VM.
388	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
389	* VERR_VMX_UNABLE_TO_START_VM or
390	* VERR_VMX_INVALID_VMCS_FIELD).
391	* @param rc The error code.
392	*/
393	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
394	{
395	AssertPtr(pVM);
396	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
397	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
398	{
399	AssertPtrReturnVoid(pVCpu);
400	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
401	}
402	pVM->hm.s.lLastError = rc;
403	}
404
405
406	/**
407	* Reads the VM-entry interruption-information field from the VMCS into the VMX
408	* transient structure.
409	*
410	* @returns VBox status code.
411	* @param pVmxTransient Pointer to the VMX transient structure.
412	*
413	* @remarks No-long-jump zone!!!
414	*/
415	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
416	{
417	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
418	AssertRCReturn(rc, rc);
419	return VINF_SUCCESS;
420	}
421
422
423	/**
424	* Reads the VM-entry exception error code field from the VMCS into
425	* the VMX transient structure.
426	*
427	* @returns VBox status code.
428	* @param pVmxTransient Pointer to the VMX transient structure.
429	*
430	* @remarks No-long-jump zone!!!
431	*/
432	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
433	{
434	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
435	AssertRCReturn(rc, rc);
436	return VINF_SUCCESS;
437	}
438
439
440	/**
441	* Reads the VM-entry exception error code field from the VMCS into
442	* the VMX transient structure.
443	*
444	* @returns VBox status code.
445	* @param pVCpu Pointer to the VMCPU.
446	* @param pVmxTransient Pointer to the VMX transient structure.
447	*
448	* @remarks No-long-jump zone!!!
449	*/
450	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
451	{
452	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
453	AssertRCReturn(rc, rc);
454	return VINF_SUCCESS;
455	}
456
457
458	/**
459	* Reads the VM-exit interruption-information field from the VMCS into the VMX
460	* transient structure.
461	*
462	* @returns VBox status code.
463	* @param pVCpu Pointer to the VMCPU.
464	* @param pVmxTransient Pointer to the VMX transient structure.
465	*/
466	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
467	{
468	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
469	{
470	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
471	AssertRCReturn(rc, rc);
472	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
473	}
474	return VINF_SUCCESS;
475	}
476
477
478	/**
479	* Reads the VM-exit interruption error code from the VMCS into the VMX
480	* transient structure.
481	*
482	* @returns VBox status code.
483	* @param pVCpu Pointer to the VMCPU.
484	* @param pVmxTransient Pointer to the VMX transient structure.
485	*/
486	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
487	{
488	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
489	{
490	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
491	AssertRCReturn(rc, rc);
492	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
493	}
494	return VINF_SUCCESS;
495	}
496
497
498	/**
499	* Reads the VM-exit instruction length field from the VMCS into the VMX
500	* transient structure.
501	*
502	* @returns VBox status code.
503	* @param pVCpu Pointer to the VMCPU.
504	* @param pVmxTransient Pointer to the VMX transient structure.
505	*/
506	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
507	{
508	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
509	{
510	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
511	AssertRCReturn(rc, rc);
512	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
513	}
514	return VINF_SUCCESS;
515	}
516
517
518	/**
519	* Reads the exit qualification from the VMCS into the VMX transient structure.
520	*
521	* @returns VBox status code.
522	* @param pVCpu Pointer to the VMCPU.
523	* @param pVmxTransient Pointer to the VMX transient structure.
524	*/
525	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
526	{
527	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
528	{
529	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
530	AssertRCReturn(rc, rc);
531	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
532	}
533	return VINF_SUCCESS;
534	}
535
536
537	/**
538	* Reads the IDT-vectoring information field from the VMCS into the VMX
539	* transient structure.
540	*
541	* @returns VBox status code.
542	* @param pVmxTransient Pointer to the VMX transient structure.
543	*
544	* @remarks No-long-jump zone!!!
545	*/
546	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
547	{
548	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
549	{
550	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
551	AssertRCReturn(rc, rc);
552	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
553	}
554	return VINF_SUCCESS;
555	}
556
557
558	/**
559	* Reads the IDT-vectoring error code from the VMCS into the VMX
560	* transient structure.
561	*
562	* @returns VBox status code.
563	* @param pVmxTransient Pointer to the VMX transient structure.
564	*/
565	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
566	{
567	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
568	{
569	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
570	AssertRCReturn(rc, rc);
571	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
572	}
573	return VINF_SUCCESS;
574	}
575
576
577	/**
578	* Enters VMX root mode operation on the current CPU.
579	*
580	* @returns VBox status code.
581	* @param pVM Pointer to the VM (optional, can be NULL, after
582	* a resume).
583	* @param HCPhysCpuPage Physical address of the VMXON region.
584	* @param pvCpuPage Pointer to the VMXON region.
585	*/
586	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
587	{
588	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
589	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
590
591	if (pVM)
592	{
593	/* Write the VMCS revision dword to the VMXON region. */
594	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
595	}
596
597	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
598	RTCCUINTREG fFlags = ASMIntDisableFlags();
599
600	/* Enable the VMX bit in CR4 if necessary. */
601	RTCCUINTREG uCr4 = ASMGetCR4();
602	if (!(uCr4 & X86_CR4_VMXE))
603	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
604
605	/* Enter VMX root mode. */
606	int rc = VMXEnable(HCPhysCpuPage); /** @todo This would #GP(0) if we are already in VMX root mode... try skip it? */
607	if (RT_FAILURE(rc))
608	ASMSetCR4(uCr4);
609
610	/* Restore interrupts. */
611	ASMSetFlags(fFlags);
612	return rc;
613	}
614
615
616	/**
617	* Exits VMX root mode operation on the current CPU.
618	*
619	* @returns VBox status code.
620	*/
621	static int hmR0VmxLeaveRootMode(void)
622	{
623	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
624	RTCCUINTREG fFlags = ASMIntDisableFlags();
625	int rc = VINF_SUCCESS;
626
627	/* If we're for some reason not in VMX root mode, then don't leave it. */
628	if (ASMGetCR4() & X86_CR4_VMXE)
629	{
630	/* Exit VMX root mode and clear the VMX bit in CR4 */
631	VMXDisable();
632	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
633	}
634	else
635	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
636
637	/* Restore interrupts. */
638	ASMSetFlags(fFlags);
639	return rc;
640	}
641
642
643	/**
644	* Allocates and maps one physically contiguous page. The allocated page is
645	* zero'd out. (Used by various VT-x structures).
646	*
647	* @returns IPRT status code.
648	* @param pMemObj Pointer to the ring-0 memory object.
649	* @param ppVirt Where to store the virtual address of the
650	* allocation.
651	* @param pPhys Where to store the physical address of the
652	* allocation.
653	*/
654	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
655	{
656	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
657	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
658	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
659
660	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
661	if (RT_FAILURE(rc))
662	return rc;
663	ppVirt = RTR0MemObjAddress(pMemObj);
664	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
665	ASMMemZero32(*ppVirt, PAGE_SIZE);
666	return VINF_SUCCESS;
667	}
668
669
670	/**
671	* Frees and unmaps an allocated physical page.
672	*
673	* @param pMemObj Pointer to the ring-0 memory object.
674	* @param ppVirt Where to re-initialize the virtual address of
675	* allocation as 0.
676	* @param pHCPhys Where to re-initialize the physical address of the
677	* allocation as 0.
678	*/
679	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
680	{
681	AssertPtr(pMemObj);
682	AssertPtr(ppVirt);
683	AssertPtr(pHCPhys);
684	if (*pMemObj != NIL_RTR0MEMOBJ)
685	{
686	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
687	AssertRC(rc);
688	*pMemObj = NIL_RTR0MEMOBJ;
689	*ppVirt = 0;
690	*pHCPhys = 0;
691	}
692	}
693
694
695	/**
696	* Worker function to free VT-x related structures.
697	*
698	* @returns IPRT status code.
699	* @param pVM Pointer to the VM.
700	*/
701	static void hmR0VmxStructsFree(PVM pVM)
702	{
703	for (VMCPUID i = 0; i < pVM->cCpus; i++)
704	{
705	PVMCPU pVCpu = &pVM->aCpus[i];
706	AssertPtr(pVCpu);
707
708	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
709	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
710
711	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
712	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
713
714	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
715	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
716	}
717
718	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
719	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
720	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
721	#endif
722	}
723
724
725	/**
726	* Worker function to allocate VT-x related VM structures.
727	*
728	* @returns IPRT status code.
729	* @param pVM Pointer to the VM.
730	*/
731	static int hmR0VmxStructsAlloc(PVM pVM)
732	{
733	/*
734	* Initialize members up-front so we can cleanup properly on allocation failure.
735	*/
736	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
737	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
738	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
739	pVM->hm.s.vmx.HCPhys##a_Name = 0;
740
741	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
742	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
743	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
744	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
745
746	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
747	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
748	#endif
749	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
750
751	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
752	for (VMCPUID i = 0; i < pVM->cCpus; i++)
753	{
754	PVMCPU pVCpu = &pVM->aCpus[i];
755	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
756	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
757	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
758	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
759	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
760	}
761	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
762	#undef VMXLOCAL_INIT_VM_MEMOBJ
763
764	/*
765	* Allocate all the VT-x structures.
766	*/
767	int rc = VINF_SUCCESS;
768	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
769	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
770	if (RT_FAILURE(rc))
771	goto cleanup;
772	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
773	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
774	#endif
775
776	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
777	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
778	{
779	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
780	&pVM->hm.s.vmx.HCPhysApicAccess);
781	if (RT_FAILURE(rc))
782	goto cleanup;
783	}
784
785	/*
786	* Initialize per-VCPU VT-x structures.
787	*/
788	for (VMCPUID i =0; i < pVM->cCpus; i++)
789	{
790	PVMCPU pVCpu = &pVM->aCpus[i];
791	AssertPtr(pVCpu);
792
793	/* Allocate the VM control structure (VMCS). */
794	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
795	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
796	if (RT_FAILURE(rc))
797	goto cleanup;
798
799	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
800	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
801	{
802	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
803	&pVCpu->hm.s.vmx.HCPhysVirtApic);
804	if (RT_FAILURE(rc))
805	goto cleanup;
806	}
807
808	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
809	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
810	{
811	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
812	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
813	if (RT_FAILURE(rc))
814	goto cleanup;
815	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
816	}
817
818	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
819	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
820	if (RT_FAILURE(rc))
821	goto cleanup;
822
823	/* Allocate the VM-exit MSR-load page for the host MSRs. */
824	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
825	if (RT_FAILURE(rc))
826	goto cleanup;
827	}
828
829	return VINF_SUCCESS;
830
831	cleanup:
832	hmR0VmxStructsFree(pVM);
833	return rc;
834	}
835
836
837	/**
838	* Does global VT-x initialization (called during module initialization).
839	*
840	* @returns VBox status code.
841	*/
842	VMMR0DECL(int) VMXR0GlobalInit(void)
843	{
844	/* Setup the main VM exit handlers. */
845	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(s_apfnVMExitHandlers));
846	#ifdef DEBUG
847	for (unsigned i = 0; i < RT_ELEMENTS(s_apfnVMExitHandlers); i++)
848	Assert(s_apfnVMExitHandlers[i]);
849	#endif
850	return VINF_SUCCESS;
851	}
852
853
854	/**
855	* Does global VT-x termination (called during module termination).
856	*/
857	VMMR0DECL(void) VMXR0GlobalTerm()
858	{
859	/* Nothing to do currently. */
860	}
861
862
863	/**
864	* Sets up and activates VT-x on the current CPU.
865	*
866	* @returns VBox status code.
867	* @param pCpu Pointer to the global CPU info struct.
868	* @param pVM Pointer to the VM (can be NULL after a host resume
869	* operation).
870	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
871	* fEnabledByHost is true).
872	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
873	* @a fEnabledByHost is true).
874	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
875	* enable VT-x/AMD-V on the host.
876	*/
877	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
878	{
879	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
880	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
881
882	if (!fEnabledByHost)
883	{
884	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
885	if (RT_FAILURE(rc))
886	return rc;
887	}
888
889	/*
890	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
891	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
892	* each time while reusing a VPID after hitting the MaxASID limit once.
893	*/
894	if ( pVM
895	&& pVM->hm.s.vmx.fVpid
896	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
897	{
898	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
899	pCpu->fFlushAsidBeforeUse = false;
900	}
901	else
902	pCpu->fFlushAsidBeforeUse = true;
903
904	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
905	++pCpu->cTlbFlushes;
906
907	return VINF_SUCCESS;
908	}
909
910
911	/**
912	* Deactivates VT-x on the current CPU.
913	*
914	* @returns VBox status code.
915	* @param pCpu Pointer to the global CPU info struct.
916	* @param pvCpuPage Pointer to the VMXON region.
917	* @param HCPhysCpuPage Physical address of the VMXON region.
918	*/
919	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
920	{
921	NOREF(pCpu);
922	NOREF(pvCpuPage);
923	NOREF(HCPhysCpuPage);
924
925	hmR0VmxLeaveRootMode();
926	return VINF_SUCCESS;
927	}
928
929
930	/**
931	* Sets the permission bits for the specified MSR in the MSR bitmap.
932	*
933	* @param pVCpu Pointer to the VMCPU.
934	* @param ulMSR The MSR value.
935	* @param enmRead Whether reading this MSR causes a VM-exit.
936	* @param enmWrite Whether writing this MSR causes a VM-exit.
937	*/
938	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, unsigned ulMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
939	{
940	unsigned ulBit;
941	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
942
943	/*
944	* Layout:
945	* 0x000 - 0x3ff - Low MSR read bits
946	* 0x400 - 0x7ff - High MSR read bits
947	* 0x800 - 0xbff - Low MSR write bits
948	* 0xc00 - 0xfff - High MSR write bits
949	*/
950	if (ulMsr <= 0x00001FFF)
951	{
952	/* Pentium-compatible MSRs */
953	ulBit = ulMsr;
954	}
955	else if ( ulMsr >= 0xC0000000
956	&& ulMsr <= 0xC0001FFF)
957	{
958	/* AMD Sixth Generation x86 Processor MSRs */
959	ulBit = (ulMsr - 0xC0000000);
960	pbMsrBitmap += 0x400;
961	}
962	else
963	{
964	AssertMsgFailed(("Invalid MSR %lx\n", ulMsr));
965	return;
966	}
967
968	Assert(ulBit <= 0x1fff);
969	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
970	ASMBitSet(pbMsrBitmap, ulBit);
971	else
972	ASMBitClear(pbMsrBitmap, ulBit);
973
974	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
975	ASMBitSet(pbMsrBitmap + 0x800, ulBit);
976	else
977	ASMBitClear(pbMsrBitmap + 0x800, ulBit);
978	}
979
980
981	/**
982	* Flushes the TLB using EPT.
983	*
984	* @returns VBox status code.
985	* @param pVM Pointer to the VM.
986	* @param pVCpu Pointer to the VMCPU.
987	* @param enmFlush Type of flush.
988	*/
989	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
990	{
991	AssertPtr(pVM);
992	Assert(pVM->hm.s.fNestedPaging);
993
994	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
995
996	uint64_t descriptor[2];
997	descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
998	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
999
1000	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1001	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
1002	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1003	}
1004
1005
1006	/**
1007	* Flushes the TLB using VPID.
1008	*
1009	* @returns VBox status code.
1010	* @param pVM Pointer to the VM.
1011	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1012	* enmFlush).
1013	* @param enmFlush Type of flush.
1014	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1015	* on @a enmFlush).
1016	*/
1017	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1018	{
1019	AssertPtr(pVM);
1020	Assert(pVM->hm.s.vmx.fVpid);
1021
1022	uint64_t descriptor[2];
1023	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1024	{
1025	descriptor[0] = 0;
1026	descriptor[1] = 0;
1027	}
1028	else
1029	{
1030	AssertPtr(pVCpu);
1031	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1032	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1033	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1034	descriptor[1] = GCPtr;
1035	}
1036
1037	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1038	AssertMsg(rc == VINF_SUCCESS,
1039	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1040	if ( RT_SUCCESS(rc)
1041	&& pVCpu)
1042	{
1043	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1044	}
1045	}
1046
1047
1048	/**
1049	* Invalidates a guest page by guest virtual address. Only relevant for
1050	* EPT/VPID, otherwise there is nothing really to invalidate.
1051	*
1052	* @returns VBox status code.
1053	* @param pVM Pointer to the VM.
1054	* @param pVCpu Pointer to the VMCPU.
1055	* @param GCVirt Guest virtual address of the page to invalidate.
1056	*/
1057	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1058	{
1059	AssertPtr(pVM);
1060	AssertPtr(pVCpu);
1061	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1062
1063	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1064	if (!fFlushPending)
1065	{
1066	/*
1067	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1068	* See @bugref{6043} and @bugref{6177}.
1069	*
1070	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1071	* function maybe called in a loop with individual addresses.
1072	*/
1073	if (pVM->hm.s.vmx.fVpid)
1074	{
1075	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1076	{
1077	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1078	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1079	}
1080	else
1081	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1082	}
1083	else if (pVM->hm.s.fNestedPaging)
1084	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1085	}
1086
1087	return VINF_SUCCESS;
1088	}
1089
1090
1091	/**
1092	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1093	* otherwise there is nothing really to invalidate.
1094	*
1095	* @returns VBox status code.
1096	* @param pVM Pointer to the VM.
1097	* @param pVCpu Pointer to the VMCPU.
1098	* @param GCPhys Guest physical address of the page to invalidate.
1099	*/
1100	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1101	{
1102	LogFlowFunc(("%RGp\n", GCPhys));
1103
1104	/*
1105	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1106	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1107	* This function might be called in a loop.
1108	*/
1109	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1110	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1111	return VINF_SUCCESS;
1112	}
1113
1114
1115	/**
1116	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1117	* case where neither EPT nor VPID is supported by the CPU.
1118	*
1119	* @param pVM Pointer to the VM.
1120	* @param pVCpu Pointer to the VMCPU.
1121	*
1122	* @remarks Called with interrupts disabled.
1123	*/
1124	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1125	{
1126	NOREF(pVM);
1127	AssertPtr(pVCpu);
1128	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1129	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1130
1131	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1132	AssertPtr(pCpu);
1133
1134	pVCpu->hm.s.TlbShootdown.cPages = 0;
1135	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1136	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1137	pVCpu->hm.s.fForceTLBFlush = false;
1138	return;
1139	}
1140
1141
1142	/**
1143	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1144	*
1145	* @param pVM Pointer to the VM.
1146	* @param pVCpu Pointer to the VMCPU.
1147	* @remarks All references to "ASID" in this function pertains to "VPID" in
1148	* Intel's nomenclature. The reason is, to avoid confusion in compare
1149	* statements since the host-CPU copies are named "ASID".
1150	*
1151	* @remarks Called with interrupts disabled.
1152	*/
1153	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1154	{
1155	AssertPtr(pVM);
1156	AssertPtr(pVCpu);
1157	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1158	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1159	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1160
1161	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1162	AssertPtr(pCpu);
1163
1164	/*
1165	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1166	* This can happen both for start & resume due to long jumps back to ring-3.
1167	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1168	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1169	*/
1170	bool fNewASID = false;
1171	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1172	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1173	{
1174	pVCpu->hm.s.fForceTLBFlush = true;
1175	fNewASID = true;
1176	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1177	}
1178
1179	/*
1180	* Check for explicit TLB shootdowns.
1181	*/
1182	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1183	{
1184	pVCpu->hm.s.fForceTLBFlush = true;
1185	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1186	}
1187
1188	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1189	if (pVCpu->hm.s.fForceTLBFlush)
1190	{
1191	if (fNewASID)
1192	{
1193	++pCpu->uCurrentAsid;
1194	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1195	{
1196	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1197	pCpu->cTlbFlushes++;
1198	pCpu->fFlushAsidBeforeUse = true;
1199	}
1200
1201	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1202	if (pCpu->fFlushAsidBeforeUse)
1203	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1204	}
1205	else
1206	{
1207	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1208	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1209	else
1210	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1211	}
1212
1213	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1214	pVCpu->hm.s.fForceTLBFlush = false;
1215	}
1216	else
1217	{
1218	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1219	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1220	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1221	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1222
1223	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1224	* not be executed. See hmQueueInvlPage() where it is commented
1225	* out. Support individual entry flushing someday. */
1226	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1227	{
1228	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1229
1230	/*
1231	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1232	* as supported by the CPU.
1233	*/
1234	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1235	{
1236	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1237	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1238	}
1239	else
1240	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1241	}
1242	else
1243	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1244	}
1245	pVCpu->hm.s.TlbShootdown.cPages = 0;
1246	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1247
1248	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1249	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1250	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1251	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1252	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1253	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1254
1255	/* Update VMCS with the VPID. */
1256	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1257	AssertRC(rc);
1258	}
1259
1260
1261	/**
1262	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1263	*
1264	* @returns VBox status code.
1265	* @param pVM Pointer to the VM.
1266	* @param pVCpu Pointer to the VMCPU.
1267	*
1268	* @remarks Called with interrupts disabled.
1269	*/
1270	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1271	{
1272	AssertPtr(pVM);
1273	AssertPtr(pVCpu);
1274	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1275	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1276
1277	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1278	AssertPtr(pCpu);
1279
1280	/*
1281	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1282	* This can happen both for start & resume due to long jumps back to ring-3.
1283	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1284	*/
1285	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1286	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1287	{
1288	pVCpu->hm.s.fForceTLBFlush = true;
1289	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1290	}
1291
1292	/* Check for explicit TLB shootdown flushes. */
1293	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1294	{
1295	pVCpu->hm.s.fForceTLBFlush = true;
1296	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1297	}
1298
1299	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1300	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1301
1302	if (pVCpu->hm.s.fForceTLBFlush)
1303	{
1304	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1305	pVCpu->hm.s.fForceTLBFlush = false;
1306	}
1307	else
1308	{
1309	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1310	* not be executed. See hmQueueInvlPage() where it is commented
1311	* out. Support individual entry flushing someday. */
1312	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1313	{
1314	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1315	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1316	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1317	}
1318	else
1319	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1320	}
1321
1322	pVCpu->hm.s.TlbShootdown.cPages= 0;
1323	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1324	}
1325
1326
1327	/**
1328	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1329	*
1330	* @returns VBox status code.
1331	* @param pVM Pointer to the VM.
1332	* @param pVCpu Pointer to the VMCPU.
1333	*
1334	* @remarks Called with interrupts disabled.
1335	*/
1336	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1337	{
1338	AssertPtr(pVM);
1339	AssertPtr(pVCpu);
1340	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1341	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1342
1343	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1344
1345	/*
1346	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1347	* This can happen both for start & resume due to long jumps back to ring-3.
1348	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1349	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1350	*/
1351	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1352	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1353	{
1354	pVCpu->hm.s.fForceTLBFlush = true;
1355	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1356	}
1357
1358	/* Check for explicit TLB shootdown flushes. */
1359	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1360	{
1361	/*
1362	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1363	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1364	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1365	*/
1366	pVCpu->hm.s.fForceTLBFlush = true;
1367	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1368	}
1369
1370	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1371	if (pVCpu->hm.s.fForceTLBFlush)
1372	{
1373	++pCpu->uCurrentAsid;
1374	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1375	{
1376	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1377	pCpu->fFlushAsidBeforeUse = true;
1378	pCpu->cTlbFlushes++;
1379	}
1380
1381	pVCpu->hm.s.fForceTLBFlush = false;
1382	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1383	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1384	if (pCpu->fFlushAsidBeforeUse)
1385	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1386	}
1387	else
1388	{
1389	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1390	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1391	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1392	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1393
1394	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1395	* not be executed. See hmQueueInvlPage() where it is commented
1396	* out. Support individual entry flushing someday. */
1397	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1398	{
1399	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1400	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1401	{
1402	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1403	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1404	}
1405	else
1406	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1407	}
1408	else
1409	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1410	}
1411
1412	pVCpu->hm.s.TlbShootdown.cPages = 0;
1413	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1414
1415	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1416	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1417	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1418	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1419	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1420	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1421
1422	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1423	AssertRC(rc);
1424	}
1425
1426
1427	/**
1428	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1429	* TLB entries from the host TLB before VM-entry.
1430	*
1431	* @returns VBox status code.
1432	* @param pVM Pointer to the VM.
1433	*/
1434	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1435	{
1436	/*
1437	* Determine optimal flush type for nested paging.
1438	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1439	* guest execution (see hmR3InitFinalizeR0()).
1440	*/
1441	if (pVM->hm.s.fNestedPaging)
1442	{
1443	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1444	{
1445	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1446	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1447	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1448	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1449	else
1450	{
1451	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1452	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1453	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1454	}
1455
1456	/* Make sure the write-back cacheable memory type for EPT is supported. */
1457	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1458	{
1459	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1460	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1461	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1462	}
1463	}
1464	else
1465	{
1466	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1467	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1468	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1469	}
1470	}
1471
1472	/*
1473	* Determine optimal flush type for VPID.
1474	*/
1475	if (pVM->hm.s.vmx.fVpid)
1476	{
1477	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1478	{
1479	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1480	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1481	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1482	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1483	else
1484	{
1485	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1486	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1487	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1488	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1489	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1490	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1491	pVM->hm.s.vmx.fVpid = false;
1492	}
1493	}
1494	else
1495	{
1496	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1497	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1498	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1499	pVM->hm.s.vmx.fVpid = false;
1500	}
1501	}
1502
1503	/*
1504	* Setup the handler for flushing tagged-TLBs.
1505	*/
1506	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1507	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbBoth;
1508	else if (pVM->hm.s.fNestedPaging)
1509	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbEpt;
1510	else if (pVM->hm.s.vmx.fVpid)
1511	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbVpid;
1512	else
1513	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbNone;
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	/**
1519	* Sets up pin-based VM-execution controls in the VMCS.
1520	*
1521	* @returns VBox status code.
1522	* @param pVM Pointer to the VM.
1523	* @param pVCpu Pointer to the VMCPU.
1524	*/
1525	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1526	{
1527	AssertPtr(pVM);
1528	AssertPtr(pVCpu);
1529
1530	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1531	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1532
1533	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1534	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1535	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1536
1537	/* Enable the VMX preemption timer. */
1538	if (pVM->hm.s.vmx.fUsePreemptTimer)
1539	{
1540	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1541	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1542	}
1543
1544	if ((val & zap) != val)
1545	{
1546	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1547	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1548	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1549	}
1550
1551	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1552	AssertRCReturn(rc, rc);
1553
1554	/* Update VCPU with the currently set pin-based VM-execution controls. */
1555	pVCpu->hm.s.vmx.u32PinCtls = val;
1556	return rc;
1557	}
1558
1559
1560	/**
1561	* Sets up processor-based VM-execution controls in the VMCS.
1562	*
1563	* @returns VBox status code.
1564	* @param pVM Pointer to the VM.
1565	* @param pVMCPU Pointer to the VMCPU.
1566	*/
1567	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1568	{
1569	AssertPtr(pVM);
1570	AssertPtr(pVCpu);
1571
1572	int rc = VERR_INTERNAL_ERROR_5;
1573	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1574	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1575
1576	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1577	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1578	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1579	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1580	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1581	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1582	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1583
1584	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1585	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1586	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1587	{
1588	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1589	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1590	}
1591
1592	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1593	if (!pVM->hm.s.fNestedPaging)
1594	{
1595	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1596	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1597	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1598	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1599	}
1600
1601	/* Use TPR shadowing if supported by the CPU. */
1602	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1603	{
1604	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1605	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1606	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1607	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1608	AssertRCReturn(rc, rc);
1609
1610	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1611	/* CR8 writes causes a VM-exit based on TPR threshold. */
1612	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1613	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1614	}
1615	else
1616	{
1617	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1618	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1619	}
1620
1621	/* Use MSR-bitmaps if supported by the CPU. */
1622	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1623	{
1624	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1625
1626	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1627	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1628	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1629	AssertRCReturn(rc, rc);
1630
1631	/*
1632	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1633	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1634	*/
1635	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1636	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1637	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1638	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1639	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1640	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1641	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1642	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1643	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1644	}
1645
1646	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1647	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1648	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1649
1650	if ((val & zap) != val)
1651	{
1652	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1653	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1654	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1655	}
1656
1657	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1658	AssertRCReturn(rc, rc);
1659
1660	/* Update VCPU with the currently set processor-based VM-execution controls. */
1661	pVCpu->hm.s.vmx.u32ProcCtls = val;
1662
1663	/*
1664	* Secondary processor-based VM-execution controls.
1665	*/
1666	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1667	{
1668	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1669	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1670
1671	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1672	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1673
1674	if (pVM->hm.s.fNestedPaging)
1675	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1676	else
1677	{
1678	/*
1679	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1680	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1681	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1682	*/
1683	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1684	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1685	}
1686
1687	if (pVM->hm.s.vmx.fVpid)
1688	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1689
1690	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1691	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1692
1693	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1694	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1695	* done dynamically. */
1696	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1697	{
1698	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1699	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1700	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1701	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1702	AssertRCReturn(rc, rc);
1703	}
1704
1705	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1706	{
1707	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1708	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1709	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1710	}
1711
1712	if ((val & zap) != val)
1713	{
1714	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1715	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1716	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1717	}
1718
1719	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1720	AssertRCReturn(rc, rc);
1721
1722	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1723	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1724	}
1725
1726	return VINF_SUCCESS;
1727	}
1728
1729
1730	/**
1731	* Sets up miscellaneous (everything other than Pin & Processor-based
1732	* VM-execution) control fields in the VMCS.
1733	*
1734	* @returns VBox status code.
1735	* @param pVM Pointer to the VM.
1736	* @param pVCpu Pointer to the VMCPU.
1737	*/
1738	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1739	{
1740	AssertPtr(pVM);
1741	AssertPtr(pVCpu);
1742
1743	int rc = VERR_GENERAL_FAILURE;
1744
1745	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1746	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1747
1748	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1749
1750	/*
1751	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1752	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1753	* We thus use the exception bitmap to control it rather than use both.
1754	*/
1755	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1756	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1757
1758	/** @todo Explore possibility of using IO-bitmaps. */
1759	/* All IO & IOIO instructions cause VM-exits. */
1760	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1761	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1762
1763	/* Setup MSR autoloading/autostoring. */
1764	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1765	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1766	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1767	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1768	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1769	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1770
1771	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1772	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1773	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1774	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1775
1776	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1777	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1778
1779	/* Setup debug controls */
1780	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1781	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1782	AssertRCReturn(rc, rc);
1783	return rc;
1784	}
1785
1786
1787	/**
1788	* Sets up the initial exception bitmap in the VMCS based on static conditions
1789	* (i.e. conditions that cannot ever change at runtime).
1790	*
1791	* @returns VBox status code.
1792	* @param pVM Pointer to the VM.
1793	* @param pVCpu Pointer to the VMCPU.
1794	*/
1795	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1796	{
1797	AssertPtr(pVM);
1798	AssertPtr(pVCpu);
1799
1800	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1801
1802	uint32_t u32XcptBitmap = 0;
1803
1804	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1805	if (!pVM->hm.s.fNestedPaging)
1806	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1807
1808	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1809	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1810	AssertRCReturn(rc, rc);
1811	return rc;
1812	}
1813
1814
1815	/**
1816	* Sets up the initial guest-state mask. The guest-state mask is consulted
1817	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1818	* for the nested virtualization case (as it would cause a VM-exit).
1819	*
1820	* @param pVCpu Pointer to the VMCPU.
1821	*/
1822	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1823	{
1824	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1825	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1826	return VINF_SUCCESS;
1827	}
1828
1829
1830	/**
1831	* Does per-VM VT-x initialization.
1832	*
1833	* @returns VBox status code.
1834	* @param pVM Pointer to the VM.
1835	*/
1836	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1837	{
1838	LogFlowFunc(("pVM=%p\n", pVM));
1839
1840	int rc = hmR0VmxStructsAlloc(pVM);
1841	if (RT_FAILURE(rc))
1842	{
1843	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1844	return rc;
1845	}
1846
1847	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1848	{
1849	PVMCPU pVCpu = &pVM->aCpus[i];
1850
1851	/* Current guest paging mode. */
1852	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1853	}
1854
1855	return VINF_SUCCESS;
1856	}
1857
1858
1859	/**
1860	* Does per-VM VT-x termination.
1861	*
1862	* @returns VBox status code.
1863	* @param pVM Pointer to the VM.
1864	*/
1865	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1866	{
1867	LogFlowFunc(("pVM=%p\n", pVM));
1868
1869	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1870	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1871	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1872	#endif
1873	hmR0VmxStructsFree(pVM);
1874	return VINF_SUCCESS;
1875	}
1876
1877
1878	/**
1879	* Sets up the VM for execution under VT-x.
1880	* This function is only called once per-VM during initalization.
1881	*
1882	* @returns VBox status code.
1883	* @param pVM Pointer to the VM.
1884	*/
1885	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1886	{
1887	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1888	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1889
1890	LogFlowFunc(("pVM=%p\n", pVM));
1891
1892	/*
1893	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1894	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1895	*/
1896	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1897	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1898	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1899	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1900	{
1901	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1902	return VERR_INTERNAL_ERROR;
1903	}
1904
1905	/* Initialize these always, see hmR3InitFinalizeR0().*/
1906	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1907	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1908
1909	/* Setup the tagged-TLB flush handlers. */
1910	int rc = hmR0VmxSetupTaggedTlb(pVM);
1911	if (RT_FAILURE(rc))
1912	{
1913	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1914	return rc;
1915	}
1916
1917	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1918	{
1919	PVMCPU pVCpu = &pVM->aCpus[i];
1920	AssertPtr(pVCpu);
1921	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1922
1923	/* Set revision dword at the beginning of the VMCS structure. */
1924	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1925
1926	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1927	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1928	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1929	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1930
1931	/* Load this VMCS as the current VMCS. */
1932	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1933	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1934	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1935
1936	/* Setup the pin-based VM-execution controls. */
1937	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1938	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1939	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1940
1941	/* Setup the processor-based VM-execution controls. */
1942	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1943	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1944	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1945
1946	/* Setup the rest (miscellaneous) VM-execution controls. */
1947	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1948	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1949	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1950
1951	/* Setup the initial exception bitmap. */
1952	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1953	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1954	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1955
1956	/* Setup the initial guest-state mask. */
1957	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1958	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1959	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1960
1961	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1962	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1963	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1964	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1965	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1966	#endif
1967
1968	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
1969	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1970	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1971	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1972
1973	/* Update the last error record for this VCPU. */
1974	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
1975	}
1976
1977	return VINF_SUCCESS;
1978	}
1979
1980
1981	/**
1982	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
1983	* the VMCS.
1984	*
1985	* @returns VBox status code.
1986	* @param pVM Pointer to the VM.
1987	* @param pVCpu Pointer to the VMCPU.
1988	*/
1989	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
1990	{
1991	RTCCUINTREG uReg = ASMGetCR0();
1992	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
1993
1994	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1995	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
1996	if (VMX_IS_64BIT_HOST_MODE())
1997	{
1998	uint64_t uReg = hmR0Get64bitCR3();
1999	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2000	}
2001	else
2002	#endif
2003	{
2004	uReg = ASMGetCR3();
2005	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2006	}
2007
2008	uReg = ASMGetCR4();
2009	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2010	AssertRCReturn(rc, rc);
2011	return rc;
2012	}
2013
2014
2015	/**
2016	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2017	* the host-state area in the VMCS.
2018	*
2019	* @returns VBox status code.
2020	* @param pVM Pointer to the VM.
2021	* @param pVCpu Pointer to the VMCPU.
2022	*/
2023	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2024	{
2025	int rc = VERR_INTERNAL_ERROR_5;
2026	RTSEL uSelCS = 0;
2027	RTSEL uSelSS = 0;
2028	RTSEL uSelDS = 0;
2029	RTSEL uSelES = 0;
2030	RTSEL uSelFS = 0;
2031	RTSEL uSelGS = 0;
2032	RTSEL uSelTR = 0;
2033
2034	/*
2035	* Host Selector registers.
2036	*/
2037	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2038	if (VMX_IS_64BIT_HOST_MODE())
2039	{
2040	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2041	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2042	}
2043	else
2044	{
2045	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2046	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2047	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2048	}
2049	#else
2050	uSelCS = ASMGetCS();
2051	uSelSS = ASMGetSS();
2052	#endif
2053
2054	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2055	/** @todo Verify if we have any platform that actually run with DS or ES with
2056	* RPL != 0 in kernel space. */
2057	uSelDS = 0;
2058	uSelES = 0;
2059	uSelFS = 0;
2060	uSelGS = 0;
2061	uSelTR = ASMGetTR();
2062
2063	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2064	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2065	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2066	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2067	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2068	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2069	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2070	Assert(uSelCS != 0);
2071	Assert(uSelTR != 0);
2072
2073	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2074	#if 0
2075	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2076	Assert(uSelSS != 0);
2077	#endif
2078
2079	/* Write these host selector fields into the host-state area in the VMCS. */
2080	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2081	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2082	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2083	#if 0
2084	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2085	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2086	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2087	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2088	#endif
2089	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2090	AssertRCReturn(rc, rc);
2091
2092	/*
2093	* Host GDTR and IDTR.
2094	*/
2095	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2096	* be safe to -not- save and restore GDTR and IDTR in the assembly
2097	* code and just do it here and don't care if the limits are zapped on
2098	* VM-exit. */
2099	RTGDTR Gdtr;
2100	RT_ZERO(Gdtr);
2101	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2102	if (VMX_IS_64BIT_HOST_MODE())
2103	{
2104	X86XDTR64 Gtr64;
2105	X86XDTR64 Idtr64;
2106	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2107	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2108	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2109	Gdtr.cbGdt = Gdtr64.cb;
2110	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2111	}
2112	else
2113	#endif
2114	{
2115	RTIDTR Idtr;
2116	ASMGetGDTR(&Gdtr);
2117	ASMGetIDTR(&Idtr);
2118	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2119	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2120	}
2121	AssertRCReturn(rc, rc);
2122
2123	/*
2124	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2125	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2126	*/
2127	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2128	{
2129	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2130	return VERR_VMX_INVALID_HOST_STATE;
2131	}
2132
2133	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2134	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2135	if (VMX_IS_64BIT_HOST_MODE())
2136	{
2137	/* We need the 64-bit TR base for hybrid darwin. */
2138	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2139	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2140	}
2141	else
2142	#endif
2143	{
2144	uintptr_t uTRBase;
2145	#if HC_ARCH_BITS == 64
2146	uTRBase = X86DESC64_BASE(pDesc);
2147	#else
2148	uTRBase = X86DESC_BASE(pDesc);
2149	#endif
2150	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2151	}
2152	AssertRCReturn(rc, rc);
2153
2154	/*
2155	* Host FS base and GS base.
2156	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2157	* would take care of the bases. In 64-bit, the MSRs come into play.
2158	*/
2159	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2160	if (VMX_IS_64BIT_HOST_MODE())
2161	{
2162	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2163	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2164	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2165	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2166	AssertRCReturn(rc, rc);
2167	}
2168	#endif
2169	return rc;
2170	}
2171
2172
2173	/**
2174	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2175	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2176	* the host after every successful VM exit.
2177	*
2178	* @returns VBox status code.
2179	* @param pVM Pointer to the VM.
2180	* @param pVCpu Pointer to the VMCPU.
2181	*/
2182	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2183	{
2184	AssertPtr(pVCpu);
2185	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2186
2187	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2188	unsigned idxHostMsr = 0;
2189	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2190
2191	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2192	{
2193	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2194	pHostMsr->u32Reserved = 0;
2195	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2196	if (CPUMIsGuestInLongMode(pVCpu))
2197	{
2198	/* Must match the EFER value in our 64 bits switcher. */
2199	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2200	}
2201	else
2202	#endif
2203	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2204	pHostMsr++; idxHostMsr++;
2205	}
2206
2207	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2208	if (VMX_IS_64BIT_HOST_MODE())
2209	{
2210	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2211	pHostMsr->u32Reserved = 0;
2212	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2213	pHostMsr++; idxHostMsr++;
2214	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2215	pHostMsr->u32Reserved = 0;
2216	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2217	pHostMsr++; idxHostMsr++;
2218	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2219	pHostMsr->u32Reserved = 0;
2220	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2221	pHostMsr++; idxHostMsr++;
2222	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2223	#if 0
2224	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2225	pMsr->u32Reserved = 0;
2226	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2227	pHostMsr++; idxHostMsr++;
2228	#endif
2229	}
2230	#endif
2231
2232	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2233	if (RT_UNLIKELY(idxHostMsr > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2234	{
2235	LogRel(("idxHostMsr=%u Cpu=%u\n", idxHostMsr, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2236	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2237	}
2238
2239	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxHostMsr);
2240
2241	/*
2242	* Host Sysenter MSRs.
2243	*/
2244	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2245	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2246	if (VMX_IS_64BIT_HOST_MODE())
2247	{
2248	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2249	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2250	}
2251	else
2252	{
2253	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2254	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2255	}
2256	#elif HC_ARCH_BITS == 32
2257	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2258	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2259	#else
2260	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2261	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2262	#endif
2263	AssertRCReturn(rc, rc);
2264
2265	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2266	* hmR0VmxSetupExitCtls() !! */
2267	return rc;
2268	}
2269
2270
2271	/**
2272	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2273	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2274	* controls".
2275	*
2276	* @returns VBox status code.
2277	* @param pVM Pointer to the VM.
2278	* @param pVCpu Pointer to the VMCPU.
2279	* @param pCtx Pointer to the guest-CPU context.
2280	*
2281	* @remarks No-long-jump zone!!!
2282	*/
2283	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2284	{
2285	int rc = VINF_SUCCESS;
2286	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2287	{
2288	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2289	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2290
2291	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2292	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2293
2294	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2295	if (CPUMIsGuestInLongModeEx(pCtx))
2296	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2297	else
2298	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2299
2300	/*
2301	* The following should not be set (since we're not in SMM mode):
2302	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2303	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2304	*/
2305
2306	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2307	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2308	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2309
2310	if ((val & zap) != val)
2311	{
2312	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2313	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2314	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2315	}
2316
2317	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2318	AssertRCReturn(rc, rc);
2319
2320	/* Update VCPU with the currently set VM-exit controls. */
2321	pVCpu->hm.s.vmx.u32EntryCtls = val;
2322	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2323	}
2324	return rc;
2325	}
2326
2327
2328	/**
2329	* Sets up the VM-exit controls in the VMCS.
2330	*
2331	* @returns VBox status code.
2332	* @param pVM Pointer to the VM.
2333	* @param pVCpu Pointer to the VMCPU.
2334	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2335	* out-of-sync. Make sure to update the required fields
2336	* before using them.
2337	*
2338	* @remarks requires EFER.
2339	*/
2340	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2341	{
2342	int rc = VINF_SUCCESS;
2343	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2344	{
2345	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2346	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2347
2348	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2349	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2350
2351	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2352	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2353	if (VMX_IS_64BIT_HOST_MODE())
2354	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2355	else
2356	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2357	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2358	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2359	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2360	else
2361	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2362	#endif
2363
2364	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2365	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2366
2367	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2368	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2369	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2370	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2371	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2372
2373	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2374	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2375
2376	if ((val & zap) != val)
2377	{
2378	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2379	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2380	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2381	}
2382
2383	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2384	AssertRCReturn(rc, rc);
2385
2386	/* Update VCPU with the currently set VM-exit controls. */
2387	pVCpu->hm.s.vmx.u32ExitCtls = val;
2388	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2389	}
2390	return rc;
2391	}
2392
2393
2394	/**
2395	* Loads the guest APIC and related state.
2396	*
2397	* @returns VBox status code.
2398	* @param pVM Pointer to the VM.
2399	* @param pVCpu Pointer to the VMCPU.
2400	* @param pCtx Pointer to the guest-CPU context.
2401	*/
2402	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2403	{
2404	int rc = VINF_SUCCESS;
2405	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2406	{
2407	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2408	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2409	{
2410	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2411
2412	bool fPendingIntr = false;
2413	uint8_t u8GuestTpr = 0;
2414	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2415	AssertRCReturn(rc, rc);
2416
2417	/*
2418	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2419	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2420	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2421	* the interrupt when we VM-exit for other reasons.
2422	*/
2423	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2424	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2425	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2426	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2427
2428	Log(("Getting TPR=%d Threshold=%u\n", pVCpu->hm.s.vmx.pbVirtApic[0x80], u32TprThreshold));
2429	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2430	AssertRCReturn(rc, rc);
2431
2432	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2433	if (pVM->hm.s.fTPRPatchingActive)
2434	{
2435	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2436	pCtx->msrLSTAR = u8GuestTpr;
2437	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2438	{
2439	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2440	if (fPendingIntr)
2441	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2442	else
2443	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2444	}
2445	}
2446	}
2447
2448	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2449	}
2450	return rc;
2451	}
2452
2453
2454	/**
2455	* Loads the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
2456	* into the guest-state area in the VMCS.
2457	*
2458	* @param pVM Pointer to the VM.
2459	* @param pVCpu Pointer to the VMCPU.
2460	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2461	* out-of-sync. Make sure to update the required fields
2462	* before using them.
2463	*
2464	* @remarks No-long-jump zone!!!
2465	*/
2466	DECLINLINE(void) hmR0VmxLoadGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2467	{
2468	/*
2469	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2470	* inhibit interrupts or clear any existing interrupt-inhibition.
2471	*/
2472	uint32_t uIntrState = 0;
2473	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2474	{
2475	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2476	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS)),
2477	("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2478	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2479	{
2480	/*
2481	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2482	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2483	* hmR0VmxInjectPendingInterrupt() relies on us clearing this flag here.
2484	*/
2485	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2486	}
2487	else if (pMixedCtx->eflags.Bits.u1IF)
2488	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2489	else
2490	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2491	}
2492
2493	Assert(!(uIntrState & 0xfffffff0)); /* Bits 31:4 MBZ. */
2494	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2495	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2496	AssertRC(rc);
2497	}
2498
2499
2500	/**
2501	* Loads the guest's RIP into the guest-state area in the VMCS.
2502	*
2503	* @returns VBox status code.
2504	* @param pVM Pointer to the VM.
2505	* @param pVCpu Pointer to the VMCPU.
2506	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2507	* out-of-sync. Make sure to update the required fields
2508	* before using them.
2509	*
2510	* @remarks No-long-jump zone!!!
2511	*/
2512	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2513	{
2514	int rc = VINF_SUCCESS;
2515	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2516	{
2517	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2518	AssertRCReturn(rc, rc);
2519	Log(("VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2520	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2521	}
2522	return rc;
2523	}
2524
2525
2526	/**
2527	* Loads the guest's RSP into the guest-state area in the VMCS.
2528	*
2529	* @returns VBox status code.
2530	* @param pVM Pointer to the VM.
2531	* @param pVCpu Pointer to the VMCPU.
2532	* @param pCtx Pointer to the guest-CPU context.
2533	*
2534	* @remarks No-long-jump zone!!!
2535	*/
2536	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2537	{
2538	int rc = VINF_SUCCESS;
2539	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2540	{
2541	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2542	AssertRCReturn(rc, rc);
2543	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2544	}
2545	return rc;
2546	}
2547
2548
2549	/**
2550	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2551	*
2552	* @returns VBox status code.
2553	* @param pVM Pointer to the VM.
2554	* @param pVCpu Pointer to the VMCPU.
2555	* @param pCtx Pointer to the guest-CPU context.
2556	*
2557	* @remarks No-long-jump zone!!!
2558	*/
2559	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2560	{
2561	int rc = VINF_SUCCESS;
2562	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2563	{
2564	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2565	Let us assert it as such and use native-width VMWRITE. */
2566	X86RFLAGS uRFlags = pCtx->rflags;
2567	Assert(uRFlags.u64 >> 32 == 0);
2568	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2569	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2570
2571	/*
2572	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2573	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2574	*/
2575	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2576	{
2577	Assert(pVM->hm.s.vmx.pRealModeTSS);
2578	Assert(PDMVmmDevHeapIsEnabled(pVM));
2579	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2580	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2581	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2582	}
2583
2584	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2585	AssertRCReturn(rc, rc);
2586
2587	Log(("VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2588	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2589	}
2590	return rc;
2591	}
2592
2593
2594	/**
2595	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2596	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2597	* assembly code.
2598	*
2599	* @returns VBox status code.
2600	* @param pVM Pointer to the VM.
2601	* @param pVCpu Pointer to the VMCPU.
2602	* @param pCtx Pointer to the guest-CPU context.
2603	*
2604	* @remarks No-long-jump zone!!!
2605	*/
2606	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2607	{
2608	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2609	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2610	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2611	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2612	return rc;
2613	}
2614
2615
2616	/**
2617	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2618	* in the VMCS.
2619	*
2620	* @returns VBox status code.
2621	* @param pVM Pointer to the VM.
2622	* @param pVCpu Pointer to the VMCPU.
2623	* @param pCtx Pointer to the guest-CPU context.
2624	*
2625	* @remarks No-long-jump zone!!!
2626	*/
2627	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2628	{
2629	int rc = VINF_SUCCESS;
2630
2631	/*
2632	* Guest CR0.
2633	* Guest FPU.
2634	*/
2635	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2636	{
2637	uint64_t u64GuestCR0 = pCtx->cr0;
2638
2639	/* The guest's view (read access) of its CR0 is unblemished. */
2640	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2641	AssertRCReturn(rc, rc);
2642	Log2(("VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2643
2644	/* Setup VT-x's view of the guest CR0. */
2645	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2646	if (pVM->hm.s.fNestedPaging)
2647	{
2648	if (CPUMIsGuestPagingEnabledEx(pCtx))
2649	{
2650	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2651	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2652	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2653	}
2654	else
2655	{
2656	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2657	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2658	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2659	}
2660
2661	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2662	AssertRCReturn(rc, rc);
2663	}
2664	else
2665	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2666
2667	/*
2668	* Guest FPU bits.
2669	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2670	* CPUs to support VT-x (prob. means all the way up to Nehalem) and no mention of with regards to UX in VM-entry checks.
2671	*/
2672	u64GuestCR0 \|= X86_CR0_NE;
2673	bool fInterceptNM = false;
2674	if (CPUMIsGuestFPUStateActive(pVCpu))
2675	{
2676	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2677	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2678	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2679	}
2680	else
2681	{
2682	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2683	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2684	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2685	}
2686
2687	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2688	bool fInterceptMF = false;
2689	if (!(pCtx->cr0 & X86_CR0_NE))
2690	fInterceptMF = true;
2691
2692	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2693	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2694	{
2695	Assert(PDMVmmDevHeapIsEnabled(pVM));
2696	Assert(pVM->hm.s.vmx.pRealModeTSS);
2697	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2698	fInterceptNM = true;
2699	fInterceptMF = true;
2700	}
2701	else
2702	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2703
2704	if (fInterceptNM)
2705	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2706	else
2707	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2708
2709	if (fInterceptMF)
2710	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2711	else
2712	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2713
2714	/* Additional intercepts for debugging, define these yourself explicitly. */
2715	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2716	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2717	\| RT_BIT(X86_XCPT_DB)
2718	\| RT_BIT(X86_XCPT_DE)
2719	\| RT_BIT(X86_XCPT_NM)
2720	\| RT_BIT(X86_XCPT_UD)
2721	\| RT_BIT(X86_XCPT_NP)
2722	\| RT_BIT(X86_XCPT_SS)
2723	\| RT_BIT(X86_XCPT_GP)
2724	\| RT_BIT(X86_XCPT_PF)
2725	\| RT_BIT(X86_XCPT_MF);
2726	#elif defined(VBOX_ALWAYS_TRAP_PF)
2727	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2728	#endif
2729
2730	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2731
2732	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2733	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2734	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2735	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2736	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2737	else
2738	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2739
2740	u64GuestCR0 \|= uSetCR0;
2741	u64GuestCR0 &= uZapCR0;
2742	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2743
2744	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2745	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2746	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2747	Log2(("VMX_VMCS_GUEST_CR0=%#RX32\n", (uint32_t)u64GuestCR0));
2748
2749	/*
2750	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2751	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2752	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2753	*/
2754	uint64_t u64CR0Mask = 0;
2755	u64CR0Mask = X86_CR0_PE
2756	\| X86_CR0_WP
2757	\| X86_CR0_PG
2758	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2759	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2760	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2761
2762	/* We don't need to intercept changes to CR0.PE with unrestricted guests. */
2763	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2764	u64CR0Mask &= ~X86_CR0_PE;
2765
2766	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2767	if (fInterceptNM)
2768	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2769	else
2770	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2771
2772	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2773	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2774	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2775	AssertRCReturn(rc, rc);
2776
2777	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2778	}
2779
2780	/*
2781	* Guest CR2.
2782	* It's always loaded in the assembler code. Nothing to do here.
2783	*/
2784
2785	/*
2786	* Guest CR3.
2787	*/
2788	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2789	{
2790	uint64_t u64GuestCR3 = 0;
2791	if (pVM->hm.s.fNestedPaging)
2792	{
2793	pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2794
2795	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2796	Assert(pVCpu->hm.s.vmx.GCPhysEPTP);
2797	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff0000000000000ULL));
2798	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2799
2800	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2801	pVCpu->hm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2802	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2803
2804	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2805	AssertMsg( ((pVCpu->hm.s.vmx.GCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2806	&& ((pVCpu->hm.s.vmx.GCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2807	("EPTP %#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2808
2809	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2810	AssertRCReturn(rc, rc);
2811	Log(("VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2812
2813	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2814	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2815	{
2816	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2817	if (CPUMIsGuestInPAEModeEx(pCtx))
2818	{
2819	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2820	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2821	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2822	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2823	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2824	AssertRCReturn(rc, rc);
2825	}
2826
2827	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2828	have Unrestricted Execution to handle the guest when it's not using paging. */
2829	u64GuestCR3 = pCtx->cr3;
2830	}
2831	else
2832	{
2833	/*
2834	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2835	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2836	* EPT takes care of translating it to host-physical addresses.
2837	*/
2838	RTGCPHYS GCPhys;
2839	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2840	Assert(PDMVmmDevHeapIsEnabled(pVM));
2841
2842	/* We obtain it here every time as the guest could have relocated this PCI region. */
2843	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2844	AssertRCReturn(rc, rc);
2845
2846	u64GuestCR3 = GCPhys;
2847	}
2848	}
2849	else
2850	{
2851	/* Non-nested paging case, just use the hypervisor's CR3. */
2852	u64GuestCR3 = PGMGetHyperCR3(pVCpu);
2853	}
2854
2855	Log2(("VMX_VMCS_GUEST_CR3=%#RX64\n", u64GuestCR3));
2856	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, u64GuestCR3);
2857	AssertRCReturn(rc, rc);
2858
2859	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2860	}
2861
2862	/*
2863	* Guest CR4.
2864	*/
2865	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2866	{
2867	uint64_t u64GuestCR4 = pCtx->cr4;
2868
2869	/* The guest's view of its CR4 is unblemished. */
2870	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2871	AssertRCReturn(rc, rc);
2872	Log2(("VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2873
2874	/* Setup VT-x's view of the guest CR4. */
2875	/*
2876	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2877	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2878	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2879	*/
2880	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2881	{
2882	Assert(pVM->hm.s.vmx.pRealModeTSS);
2883	Assert(PDMVmmDevHeapIsEnabled(pVM));
2884	u64GuestCR4 &= ~X86_CR4_VME;
2885	}
2886
2887	if (pVM->hm.s.fNestedPaging)
2888	{
2889	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2890	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2891	{
2892	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2893	u64GuestCR4 \|= X86_CR4_PSE;
2894	/* Our identity mapping is a 32 bits page directory. */
2895	u64GuestCR4 &= ~X86_CR4_PAE;
2896	}
2897	/* else use guest CR4.*/
2898	}
2899	else
2900	{
2901	/*
2902	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2903	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2904	*/
2905	switch (pVCpu->hm.s.enmShadowMode)
2906	{
2907	case PGMMODE_REAL: /* Real-mode. */
2908	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2909	case PGMMODE_32_BIT: /* 32-bit paging. */
2910	{
2911	u64GuestCR4 &= ~X86_CR4_PAE;
2912	break;
2913	}
2914
2915	case PGMMODE_PAE: /* PAE paging. */
2916	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2917	{
2918	u64GuestCR4 \|= X86_CR4_PAE;
2919	break;
2920	}
2921
2922	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2923	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2924	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2925	break;
2926	#endif
2927	default:
2928	AssertFailed();
2929	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2930	}
2931	}
2932
2933	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2934	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2935	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2936	u64GuestCR4 \|= uSetCR4;
2937	u64GuestCR4 &= uZapCR4;
2938
2939	/* Write VT-x's view of the guest CR4 into the VMCS. */
2940	Log2(("VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2941	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2942
2943	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2944	uint64_t u64CR4Mask = 0;
2945	u64CR4Mask = X86_CR4_VME
2946	\| X86_CR4_PAE
2947	\| X86_CR4_PGE
2948	\| X86_CR4_PSE
2949	\| X86_CR4_VMXE;
2950	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
2951	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
2952	AssertRCReturn(rc, rc);
2953
2954	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
2955	}
2956	return rc;
2957	}
2958
2959
2960	/**
2961	* Loads the guest debug registers into the guest-state area in the VMCS.
2962	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
2963	*
2964	* @returns VBox status code.
2965	* @param pVM Pointer to the VM.
2966	* @param pVCpu Pointer to the VMCPU.
2967	* @param pCtx Pointer to the guest-CPU context.
2968	*
2969	* @remarks No-long-jump zone!!!
2970	*/
2971	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2972	{
2973	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
2974	return VINF_SUCCESS;
2975
2976	#ifdef DEBUG
2977	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
2978	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
2979	{
2980	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
2981	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
2982	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
2983	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
2984	}
2985	#endif
2986
2987	int rc = VERR_INTERNAL_ERROR_5;
2988	bool fInterceptDB = false;
2989	bool fInterceptMovDRx = false;
2990	if (DBGFIsStepping(pVCpu))
2991	{
2992	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
2993	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2994	{
2995	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2996	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2997	AssertRCReturn(rc, rc);
2998	Assert(fInterceptDB == false);
2999	}
3000	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
3001	{
3002	Assert(fInterceptDB == false);
3003	/** @todo can we somehow signal DBGF that it cannot single-step instead of
3004	* just continuing? */
3005	}
3006	else
3007	fInterceptDB = true;
3008	}
3009	else
3010	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
3011
3012	/*
3013	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
3014	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
3015	* The same for the hypervisor DRx registers, priority is for the guest here.
3016	*/
3017	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3018	&& !CPUMIsGuestDebugStateActive(pVCpu))
3019	{
3020	/* Save the host and load the guest debug registers. This will make the guest debug state active. */
3021	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3022	AssertRC(rc);
3023	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3024	Assert(fInterceptMovDRx == false);
3025	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3026	}
3027	else if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)
3028	&& !CPUMIsHyperDebugStateActive(pVCpu))
3029	{
3030	/* Save the host and load the hypervisor debug registers. This will make the hyper debug state active. */
3031	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3032	AssertRC(rc);
3033	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3034	fInterceptMovDRx = true;
3035	}
3036	else
3037	Assert(fInterceptMovDRx == false); /* No need to intercept MOV DRx if DBGF is not active nor the guest is debugging. */
3038
3039	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3040	if (fInterceptDB)
3041	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3042	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3043	{
3044	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3045	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3046	#endif
3047	}
3048
3049	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3050	if (fInterceptMovDRx)
3051	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3052	else
3053	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3054
3055	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3056	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3057
3058	/* The guest's view of its DR7 is unblemished. */
3059	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3060
3061	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3062	return rc;
3063	}
3064
3065
3066	#ifdef DEBUG
3067	/**
3068	* Debug function to validate segment registers.
3069	*/
3070	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3071	{
3072	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3073	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3074	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3075	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3076	{
3077	/* Protected mode checks */
3078	/* CS */
3079	Assert(pCtx->cs.Attr.n.u1Present);
3080	Assert(!(pCtx->cs.Attr.u & 0xf00));
3081	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3082	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3083	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3084	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3085	\|\| (pCtx->cs.Attr.n.u1Granularity));
3086	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3087	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3088	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3089	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3090	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3091	else
3092	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3093	/* SS */
3094	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3095	{
3096	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3097	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3098	Assert(pCtx->ss.Attr.n.u1Present);
3099	Assert(!(pCtx->ss.Attr.u & 0xf00));
3100	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3101	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3102	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3103	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3104	\|\| (pCtx->ss.Attr.n.u1Granularity));
3105	}
3106	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3107	/* CR0 might not be up-to-date here always, hence disabled. */
3108	#if 0
3109	if (!pCtx->cr0 & X86_CR0_PE)
3110	Assert(!pCtx->ss.Attr.n.u2Dpl);
3111	#endif
3112	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3113	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3114	{
3115	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3116	Assert(pCtx->ds.Attr.n.u1Present);
3117	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3118	Assert(!(pCtx->ds.Attr.u & 0xf00));
3119	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3120	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3121	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3122	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3123	\|\| (pCtx->ds.Attr.n.u1Granularity));
3124	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3125	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3126	}
3127	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3128	{
3129	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3130	Assert(pCtx->es.Attr.n.u1Present);
3131	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3132	Assert(!(pCtx->es.Attr.u & 0xf00));
3133	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3134	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3135	\|\| !(pCtx->es.Attr.n.u1Granularity));
3136	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3137	\|\| (pCtx->es.Attr.n.u1Granularity));
3138	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3139	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3140	}
3141	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3142	{
3143	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3144	Assert(pCtx->fs.Attr.n.u1Present);
3145	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3146	Assert(!(pCtx->fs.Attr.u & 0xf00));
3147	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3148	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3149	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3150	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3151	\|\| (pCtx->fs.Attr.n.u1Granularity));
3152	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3153	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3154	}
3155	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3156	{
3157	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3158	Assert(pCtx->gs.Attr.n.u1Present);
3159	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3160	Assert(!(pCtx->gs.Attr.u & 0xf00));
3161	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3162	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3163	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3164	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3165	\|\| (pCtx->gs.Attr.n.u1Granularity));
3166	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3167	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3168	}
3169	/* 64-bit capable CPUs. */
3170	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3171	Assert(!(pCtx->cs.u64Base >> 32));
3172	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3173	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3174	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3175	# endif
3176	}
3177	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3178	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3179	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3180	{
3181	/* Real and v86 mode checks. */
3182	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3183	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3184	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3185	{
3186	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3187	}
3188	else
3189	{
3190	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3191	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3192	}
3193
3194	/* CS */
3195	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3196	Assert(pCtx->cs.u32Limit == 0xffff);
3197	Assert(u32CSAttr == 0xf3);
3198	/* SS */
3199	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3200	Assert(pCtx->ss.u32Limit == 0xffff);
3201	Assert(u32SSAttr == 0xf3);
3202	/* DS */
3203	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3204	Assert(pCtx->ds.u32Limit == 0xffff);
3205	Assert(u32DSAttr == 0xf3);
3206	/* ES */
3207	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3208	Assert(pCtx->es.u32Limit == 0xffff);
3209	Assert(u32ESAttr == 0xf3);
3210	/* FS */
3211	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3212	Assert(pCtx->fs.u32Limit == 0xffff);
3213	Assert(u32FSAttr == 0xf3);
3214	/* GS */
3215	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3216	Assert(pCtx->gs.u32Limit == 0xffff);
3217	Assert(u32GSAttr == 0xf3);
3218	/* 64-bit capable CPUs. */
3219	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3220	Assert(!(pCtx->cs.u64Base >> 32));
3221	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3222	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3223	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3224	# endif
3225	}
3226	}
3227	#endif /* DEBUG */
3228
3229
3230	/**
3231	* Writes a guest segment register into the guest-state area in the VMCS.
3232	*
3233	* @returns VBox status code.
3234	* @param pVM Pointer to the VM.
3235	* @param pVCpu Pointer to the VMCPU.
3236	* @param idxSel Index of the selector in the VMCS.
3237	* @param idxLimit Index of the segment limit in the VMCS.
3238	* @param idxBase Index of the segment base in the VMCS.
3239	* @param idxAccess Index of the access rights of the segment in the VMCS.
3240	* @param pSelReg Pointer to the segment selector.
3241	* @param pCtx Pointer to the guest-CPU context.
3242	*
3243	* @remarks No-long-jump zone!!!
3244	*/
3245	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3246	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3247	{
3248	int rc;
3249	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3250	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3251	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3252	AssertRCReturn(rc, rc);
3253
3254	uint32_t u32Access = pSelReg->Attr.u;
3255	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3256	{
3257	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3258	u32Access = 0xf3;
3259	Assert(pVM->hm.s.vmx.pRealModeTSS);
3260	Assert(PDMVmmDevHeapIsEnabled(pVM));
3261	}
3262	else
3263	{
3264	/*
3265	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3266	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3267	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3268	* loaded in protected-mode have their attribute as 0.
3269	*/
3270	if (!u32Access)
3271	u32Access = VMX_SEL_UNUSABLE;
3272	}
3273
3274	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3275	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3276	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3277
3278	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3279	AssertRCReturn(rc, rc);
3280	return rc;
3281	}
3282
3283
3284	/**
3285	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3286	* into the guest-state area in the VMCS.
3287	*
3288	* @returns VBox status code.
3289	* @param pVM Pointer to the VM.
3290	* @param pVCPU Pointer to the VMCPU.
3291	* @param pCtx Pointer to the guest-CPU context.
3292	*
3293	* @remarks No-long-jump zone!!!
3294	* @remarks Requires RFLAGS (for debug assertions).
3295	*/
3296	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3297	{
3298	int rc = VERR_INTERNAL_ERROR_5;
3299
3300	/*
3301	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3302	*/
3303	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3304	{
3305	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3306	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3307	{
3308	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3309	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3310	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3311	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3312	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3313	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3314	}
3315
3316	#ifdef VBOX_WITH_REM
3317	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3318	{
3319	Assert(pVM->hm.s.vmx.pRealModeTSS);
3320	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3321	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3322	{
3323	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3324	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3325	&& enmGuestMode >= PGMMODE_PROTECTED)
3326	{
3327	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3328	in real-mode (e.g. OpenBSD 4.0) */
3329	REMFlushTBs(pVM);
3330	Log2(("Switch to protected mode detected!\n"));
3331	}
3332	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3333	}
3334	}
3335	#endif
3336	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3337	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3338	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3339	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3340	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3341	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3342	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3343	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3344	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3345	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3346	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3347	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3348	AssertRCReturn(rc, rc);
3349
3350	#ifdef DEBUG
3351	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3352	#endif
3353	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3354	}
3355
3356	/*
3357	* Guest TR.
3358	*/
3359	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3360	{
3361	/*
3362	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3363	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3364	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3365	*/
3366	uint16_t u16Sel = 0;
3367	uint32_t u32Limit = 0;
3368	uint64_t u64Base = 0;
3369	uint32_t u32AccessRights = 0;
3370
3371	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3372	{
3373	u16Sel = pCtx->tr.Sel;
3374	u32Limit = pCtx->tr.u32Limit;
3375	u64Base = pCtx->tr.u64Base;
3376	u32AccessRights = pCtx->tr.Attr.u;
3377	}
3378	else
3379	{
3380	Assert(pVM->hm.s.vmx.pRealModeTSS);
3381	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3382
3383	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3384	RTGCPHYS GCPhys;
3385	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3386	AssertRCReturn(rc, rc);
3387
3388	X86DESCATTR DescAttr;
3389	DescAttr.u = 0;
3390	DescAttr.n.u1Present = 1;
3391	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3392
3393	u16Sel = 0;
3394	u32Limit = HM_VTX_TSS_SIZE;
3395	u64Base = GCPhys; /* in real-mode phys = virt. */
3396	u32AccessRights = DescAttr.u;
3397	}
3398
3399	/* Validate. */
3400	Assert(!(u16Sel & RT_BIT(2)));
3401	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3402	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3403	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3404	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3405	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3406	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3407	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3408	Assert( (u32Limit & 0xfff) == 0xfff
3409	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3410	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3411	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3412
3413	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3414	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3415	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3416	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3417	AssertRCReturn(rc, rc);
3418
3419	Log2(("VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3420	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3421	}
3422
3423	/*
3424	* Guest GDTR.
3425	*/
3426	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3427	{
3428	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3429	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3430	AssertRCReturn(rc, rc);
3431
3432	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3433	Log2(("VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3434	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3435	}
3436
3437	/*
3438	* Guest LDTR.
3439	*/
3440	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3441	{
3442	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3443	uint32_t u32Access = 0;
3444	if (!(pCtx->ldtr.Attr.u & VMX_SEL_UNUSABLE))
3445	u32Access = VMX_SEL_UNUSABLE;
3446	else
3447	u32Access = pCtx->ldtr.Attr.u;
3448
3449	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3450	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3451	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3452	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3453	AssertRCReturn(rc, rc);
3454
3455	/* Validate. */
3456	if (!(u32Access & VMX_SEL_UNUSABLE))
3457	{
3458	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3459	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3460	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3461	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3462	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3463	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3464	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3465	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3466	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3467	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3468	}
3469
3470	Log2(("VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3471	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3472	}
3473
3474	/*
3475	* Guest IDTR.
3476	*/
3477	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3478	{
3479	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3480	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3481	AssertRCReturn(rc, rc);
3482
3483	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3484	Log2(("VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3485	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3486	}
3487
3488	/*
3489	* Guest FS & GS base MSRs.
3490	* We already initialized the FS & GS base as part of the guest segment registers, but the guest's FS/GS base
3491	* MSRs might have changed (e.g. due to WRMSR) and we need to update the bases if that happened. These MSRs
3492	* are only available in 64-bit mode.
3493	*/
3494	/** @todo Avoid duplication of this code in assembly (see MYPUSHSEGS) - it
3495	* should not be necessary to do it in assembly again. */
3496	if (CPUMIsGuestInLongModeEx(pCtx))
3497	{
3498	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FS_BASE_MSR)
3499	{
3500	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
3501	AssertRCReturn(rc, rc);
3502	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_FS_BASE_MSR;
3503	}
3504
3505	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GS_BASE_MSR)
3506	{
3507	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
3508	AssertRCReturn(rc, rc);
3509	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GS_BASE_MSR;
3510	}
3511	}
3512	else
3513	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_FS_BASE_MSR \| HM_CHANGED_GUEST_GS_BASE_MSR);
3514
3515	return VINF_SUCCESS;
3516	}
3517
3518
3519	/**
3520	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3521	* areas. These MSRs will automatically be loaded to the host CPU on every
3522	* successful VM entry and stored from the host CPU on every successful VM exit.
3523	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3524	*
3525	* @returns VBox status code.
3526	* @param pVM Pointer to the VM.
3527	* @param pVCpu Pointer to the VMCPU.
3528	* @param pCtx Pointer to the guest-CPU context.
3529	*
3530	* @remarks No-long-jump zone!!!
3531	*/
3532	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3533	{
3534	AssertPtr(pVCpu);
3535	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3536
3537	/*
3538	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3539	*/
3540	int rc = VINF_SUCCESS;
3541	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3542	{
3543	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3544	unsigned cGuestMsrs = 0;
3545
3546	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3547	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3548	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3549	if (fSupportsNX \|\| fSupportsLongMode)
3550	{
3551	/** @todo support save IA32_EFER, i.e.
3552	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3553	* guest EFER need not be part of the VM-entry MSR-load area. */
3554	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3555	pGuestMsr->u32Reserved = 0;
3556	pGuestMsr->u64Value = pCtx->msrEFER;
3557	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3558	if (!CPUMIsGuestInLongModeEx(pCtx))
3559	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3560	pGuestMsr++; cGuestMsrs++;
3561	if (fSupportsLongMode)
3562	{
3563	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3564	pGuestMsr->u32Reserved = 0;
3565	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3566	pGuestMsr++; cGuestMsrs++;
3567	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3568	pGuestMsr->u32Reserved = 0;
3569	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3570	pGuestMsr++; cGuestMsrs++;
3571	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3572	pGuestMsr->u32Reserved = 0;
3573	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3574	pGuestMsr++; cGuestMsrs++;
3575	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3576	#if 0
3577	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3578	pGuestMsr->u32Reserved = 0;
3579	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3580	pGuestMsr++; cGuestMsrs++;
3581	#endif
3582	}
3583	}
3584
3585	/*
3586	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3587	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3588	*/
3589	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3590	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3591	{
3592	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3593	pGuestMsr->u32Reserved = 0;
3594	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3595	AssertRCReturn(rc, rc);
3596	pGuestMsr++; cGuestMsrs++;
3597	}
3598
3599	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3600	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3601	{
3602	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3603	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3604	}
3605
3606	/* Update the VCPU's copy of the guest MSR count. */
3607	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3608	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3609	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3610	AssertRCReturn(rc, rc);
3611
3612	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3613	}
3614
3615	/*
3616	* Guest Sysenter MSRs.
3617	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3618	* VM exits on WRMSRs for these MSRs.
3619	*/
3620	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3621	{
3622	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3623	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3624	}
3625	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3626	{
3627	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3628	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3629	}
3630	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3631	{
3632	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3633	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3634	}
3635	AssertRCReturn(rc, rc);
3636
3637	return rc;
3638	}
3639
3640
3641	/**
3642	* Loads the guest activity state into the guest-state area in the VMCS.
3643	*
3644	* @returns VBox status code.
3645	* @param pVM Pointer to the VM.
3646	* @param pVCpu Pointer to the VMCPU.
3647	* @param pCtx Pointer to the guest-CPU context.
3648	*
3649	* @remarks No-long-jump zone!!!
3650	*/
3651	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3652	{
3653	/** @todo See if we can make use of other states, e.g.
3654	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3655	int rc = VINF_SUCCESS;
3656	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3657	{
3658	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3659	AssertRCReturn(rc, rc);
3660	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3661	}
3662	return rc;
3663	}
3664
3665
3666	/**
3667	* Sets up the appropriate function to run guest code.
3668	*
3669	* @returns VBox status code.
3670	* @param pVM Pointer to the VM.
3671	* @param pVCpu Pointer to the VMCPU.
3672	* @param pCtx Pointer to the guest-CPU context.
3673	*
3674	* @remarks No-long-jump zone!!!
3675	*/
3676	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3677	{
3678	if (CPUMIsGuestInLongModeEx(pCtx))
3679	{
3680	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3681	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3682	#endif
3683	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3684	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3685	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3686	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3687	#else
3688	/* 64-bit host or hybrid host. */
3689	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3690	#endif
3691	}
3692	else
3693	{
3694	/* Guest is not in long mode, use the 32-bit handler. */
3695	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3696	}
3697	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3698	return VINF_SUCCESS;
3699	}
3700
3701
3702	/**
3703	* Wrapper for running the guest code in VT-x.
3704	*
3705	* @returns VBox strict status code.
3706	* @param pVM Pointer to the VM.
3707	* @param pVCpu Pointer to the VMCPU.
3708	* @param pCtx Pointer to the guest-CPU context.
3709	*
3710	* @remarks No-long-jump zone!!!
3711	*/
3712	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3713	{
3714	/*
3715	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3716	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3717	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3718	*/
3719	#ifdef VBOX_WITH_KERNEL_USING_XMM
3720	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3721	#else
3722	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3723	#endif
3724	}
3725
3726
3727	/**
3728	* Report world-switch error and dump some useful debug info.
3729	*
3730	* @param pVM Pointer to the VM.
3731	* @param pVCpu Pointer to the VMCPU.
3732	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3733	* @param pCtx Pointer to the guest-CPU context.
3734	* @param pVmxTransient Pointer to the VMX transient structure (only
3735	* exitReason updated).
3736	*/
3737	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3738	{
3739	Assert(pVM);
3740	Assert(pVCpu);
3741	Assert(pCtx);
3742	Assert(pVmxTransient);
3743	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3744
3745	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3746	switch (rcVMRun)
3747	{
3748	case VERR_VMX_INVALID_VMXON_PTR:
3749	AssertFailed();
3750	break;
3751	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3752	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3753	{
3754	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3755	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3756	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3757	AssertRC(rc);
3758
3759	#ifdef VBOX_STRICT
3760	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3761	pVmxTransient->uExitReason));
3762	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3763	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3764	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3765	Log(("InstrError Desc. \"%s\"\n", s_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3766	else
3767	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3768
3769	/* VMX control bits. */
3770	uint32_t u32Val;
3771	uint64_t u64Val;
3772	RTHCUINTREG uHCReg;
3773	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3774	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3775	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3776	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3777	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3778	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3779	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3780	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3781	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3782	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3783	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3784	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3785	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3786	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3787	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3788	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3789	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3790	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3791	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3792	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3793	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3794	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3795	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3796	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3797	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3798	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3799	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3800	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3801	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3802	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3803	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3804	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3805	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3806	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3807	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3808	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3809	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3810	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3811	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3812	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3813	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3814	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3815
3816	/* Guest bits. */
3817	RTGCUINTREG uGCReg;
3818	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3819	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3820	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3821	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3822	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3823	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3824	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3825	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3826
3827	/* Host bits. */
3828	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3829	Log(("Host CR0 %#RHr\n", uHCReg));
3830	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3831	Log(("Host CR3 %#RHr\n", uHCReg));
3832	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3833	Log(("Host CR4 %#RHr\n", uHCReg));
3834
3835	RTGDTR HostGdtr;
3836	PCX86DESCHC pDesc;
3837	ASMGetGDTR(&HostGdtr);
3838	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3839	Log(("Host CS %#08x\n", u32Val));
3840	if (u32Val < HostGdtr.cbGdt)
3841	{
3842	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3843	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3844	}
3845
3846	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3847	Log(("Host DS %#08x\n", u32Val));
3848	if (u32Val < HostGdtr.cbGdt)
3849	{
3850	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3851	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3852	}
3853
3854	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3855	Log(("Host ES %#08x\n", u32Val));
3856	if (u32Val < HostGdtr.cbGdt)
3857	{
3858	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3859	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3860	}
3861
3862	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3863	Log(("Host FS %#08x\n", u32Val));
3864	if (u32Val < HostGdtr.cbGdt)
3865	{
3866	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3867	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3868	}
3869
3870	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3871	Log(("Host GS %#08x\n", u32Val));
3872	if (u32Val < HostGdtr.cbGdt)
3873	{
3874	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3875	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3876	}
3877
3878	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3879	Log(("Host SS %#08x\n", u32Val));
3880	if (u32Val < HostGdtr.cbGdt)
3881	{
3882	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3883	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3884	}
3885
3886	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3887	Log(("Host TR %#08x\n", u32Val));
3888	if (u32Val < HostGdtr.cbGdt)
3889	{
3890	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3891	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3892	}
3893
3894	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3895	Log(("Host TR Base %#RHv\n", uHCReg));
3896	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3897	Log(("Host GDTR Base %#RHv\n", uHCReg));
3898	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3899	Log(("Host IDTR Base %#RHv\n", uHCReg));
3900	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3901	Log(("Host SYSENTER CS %#08x\n", u32Val));
3902	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3903	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3904	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3905	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3906	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3907	Log(("Host RSP %#RHv\n", uHCReg));
3908	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3909	Log(("Host RIP %#RHv\n", uHCReg));
3910	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3911	if (VMX_IS_64BIT_HOST_MODE())
3912	{
3913	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3914	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3915	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3916	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3917	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3918	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3919	}
3920	# endif
3921	#endif /* VBOX_STRICT */
3922	break;
3923	}
3924
3925	default:
3926	/* Impossible */
3927	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3928	break;
3929	}
3930	NOREF(pVM);
3931	}
3932
3933
3934	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3935	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3936	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3937	#endif
3938
3939	#ifdef VBOX_STRICT
3940	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3941	{
3942	switch (idxField)
3943	{
3944	case VMX_VMCS_GUEST_RIP:
3945	case VMX_VMCS_GUEST_RSP:
3946	case VMX_VMCS_GUEST_RFLAGS:
3947	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
3948	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
3949	case VMX_VMCS_GUEST_CR0:
3950	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
3951	case VMX_VMCS_GUEST_CR4:
3952	case VMX_VMCS_GUEST_DR7:
3953	case VMX_VMCS32_GUEST_SYSENTER_CS:
3954	case VMX_VMCS_GUEST_SYSENTER_EIP:
3955	case VMX_VMCS_GUEST_SYSENTER_ESP:
3956	case VMX_VMCS32_GUEST_GDTR_LIMIT:
3957	case VMX_VMCS_GUEST_GDTR_BASE:
3958	case VMX_VMCS32_GUEST_IDTR_LIMIT:
3959	case VMX_VMCS_GUEST_IDTR_BASE:
3960	case VMX_VMCS16_GUEST_FIELD_CS:
3961	case VMX_VMCS32_GUEST_CS_LIMIT:
3962	case VMX_VMCS_GUEST_CS_BASE:
3963	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
3964	case VMX_VMCS16_GUEST_FIELD_DS:
3965	case VMX_VMCS32_GUEST_DS_LIMIT:
3966	case VMX_VMCS_GUEST_DS_BASE:
3967	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
3968	case VMX_VMCS16_GUEST_FIELD_ES:
3969	case VMX_VMCS32_GUEST_ES_LIMIT:
3970	case VMX_VMCS_GUEST_ES_BASE:
3971	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
3972	case VMX_VMCS16_GUEST_FIELD_FS:
3973	case VMX_VMCS32_GUEST_FS_LIMIT:
3974	case VMX_VMCS_GUEST_FS_BASE:
3975	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
3976	case VMX_VMCS16_GUEST_FIELD_GS:
3977	case VMX_VMCS32_GUEST_GS_LIMIT:
3978	case VMX_VMCS_GUEST_GS_BASE:
3979	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
3980	case VMX_VMCS16_GUEST_FIELD_SS:
3981	case VMX_VMCS32_GUEST_SS_LIMIT:
3982	case VMX_VMCS_GUEST_SS_BASE:
3983	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
3984	case VMX_VMCS16_GUEST_FIELD_LDTR:
3985	case VMX_VMCS32_GUEST_LDTR_LIMIT:
3986	case VMX_VMCS_GUEST_LDTR_BASE:
3987	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
3988	case VMX_VMCS16_GUEST_FIELD_TR:
3989	case VMX_VMCS32_GUEST_TR_LIMIT:
3990	case VMX_VMCS_GUEST_TR_BASE:
3991	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
3992	case VMX_VMCS32_RO_EXIT_REASON:
3993	case VMX_VMCS32_RO_VM_INSTR_ERROR:
3994	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
3995	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
3996	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
3997	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
3998	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3999	case VMX_VMCS32_RO_IDT_INFO:
4000	case VMX_VMCS32_RO_IDT_ERROR_CODE:
4001	case VMX_VMCS_GUEST_CR3:
4002	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4003	return true;
4004	}
4005	return false;
4006	}
4007
4008	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4009	{
4010	switch (idxField)
4011	{
4012	case VMX_VMCS_GUEST_LDTR_BASE:
4013	case VMX_VMCS_GUEST_TR_BASE:
4014	case VMX_VMCS_GUEST_GDTR_BASE:
4015	case VMX_VMCS_GUEST_IDTR_BASE:
4016	case VMX_VMCS_GUEST_SYSENTER_EIP:
4017	case VMX_VMCS_GUEST_SYSENTER_ESP:
4018	case VMX_VMCS_GUEST_CR0:
4019	case VMX_VMCS_GUEST_CR4:
4020	case VMX_VMCS_GUEST_CR3:
4021	case VMX_VMCS_GUEST_DR7:
4022	case VMX_VMCS_GUEST_RIP:
4023	case VMX_VMCS_GUEST_RSP:
4024	case VMX_VMCS_GUEST_CS_BASE:
4025	case VMX_VMCS_GUEST_DS_BASE:
4026	case VMX_VMCS_GUEST_ES_BASE:
4027	case VMX_VMCS_GUEST_FS_BASE:
4028	case VMX_VMCS_GUEST_GS_BASE:
4029	case VMX_VMCS_GUEST_SS_BASE:
4030	return true;
4031	}
4032	return false;
4033	}
4034	#endif /* VBOX_STRICT */
4035
4036	/**
4037	* Executes the specified handler in 64-bit mode.
4038	*
4039	* @returns VBox status code.
4040	* @param pVM Pointer to the VM.
4041	* @param pVCpu Pointer to the VMCPU.
4042	* @param pCtx Pointer to the guest CPU context.
4043	* @param pfnHandler Pointer to the RC handler function.
4044	* @param cbParam Number of parameters.
4045	* @param paParam Array of 32-bit parameters.
4046	*/
4047	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
4048	uint32_t *paParam)
4049	{
4050	int rc, rc2;
4051	PHMGLOBLCPUINFO pCpu;
4052	RTHCPHYS HCPhysCpuPage;
4053	RTHCUINTREG uOldEFlags;
4054
4055	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4056	Assert(pfnHandler);
4057	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4058	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4059
4060	#ifdef VBOX_STRICT
4061	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4062	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4063
4064	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4065	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4066	#endif
4067
4068	/* Disable interrupts. */
4069	uOldEFlags = ASMIntDisableFlags();
4070
4071	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4072	RTCPUID idHostCpu = RTMpCpuId();
4073	CPUMR0SetLApic(pVM, idHostCpu);
4074	#endif
4075
4076	pCpu = HMR0GetCurrentCpu();
4077	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4078
4079	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4080	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4081
4082	/* Leave VMX Root Mode. */
4083	VMXDisable();
4084
4085	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4086
4087	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4088	CPUMSetHyperEIP(pVCpu, pfnHandler);
4089	for (int i = (int)cbParam - 1; i >= 0; i--)
4090	CPUMPushHyper(pVCpu, paParam[i]);
4091
4092	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4093
4094	/* Call the switcher. */
4095	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4096	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4097
4098	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4099	/* Make sure the VMX instructions don't cause #UD faults. */
4100	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4101
4102	/* Re-enter VMX Root Mode */
4103	rc2 = VMXEnable(HCPhysCpuPage);
4104	if (RT_FAILURE(rc2))
4105	{
4106	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4107	ASMSetFlags(uOldEFlags);
4108	return rc2;
4109	}
4110
4111	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4112	AssertRC(rc2);
4113	Assert(!(ASMGetFlags() & X86_EFL_IF));
4114	ASMSetFlags(uOldEFlags);
4115	return rc;
4116	}
4117
4118
4119	/**
4120	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4121	* supporting 64-bit guests.
4122	*
4123	* @returns VBox status code.
4124	* @param fResume Whether to VMLAUNCH or VMRESUME.
4125	* @param pCtx Pointer to the guest-CPU context.
4126	* @param pCache Pointer to the VMCS cache.
4127	* @param pVM Pointer to the VM.
4128	* @param pVCpu Pointer to the VMCPU.
4129	*/
4130	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4131	{
4132	uint32_t aParam[6];
4133	PHMGLOBLCPUINFO pCpu = NULL;
4134	RTHCPHYS HCPhysCpuPage = 0;
4135	int rc = VERR_INTERNAL_ERROR_5;
4136
4137	pCpu = HMR0GetCurrentCpu();
4138	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4139
4140	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4141	pCache->uPos = 1;
4142	pCache->interPD = PGMGetInterPaeCR3(pVM);
4143	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4144	#endif
4145
4146	#ifdef DEBUG
4147	pCache->TestIn.HCPhysCpuPage= 0;
4148	pCache->TestIn.HCPhysVmcs = 0;
4149	pCache->TestIn.pCache = 0;
4150	pCache->TestOut.HCPhysVmcs = 0;
4151	pCache->TestOut.pCache = 0;
4152	pCache->TestOut.pCtx = 0;
4153	pCache->TestOut.eflags = 0;
4154	#endif
4155
4156	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4157	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4158	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4159	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4160	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4161	aParam[5] = 0;
4162
4163	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4164	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4165	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4166	#endif
4167	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4168
4169	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4170	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4171	Assert(pCtx->dr[4] == 10);
4172	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4173	#endif
4174
4175	#ifdef DEBUG
4176	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4177	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4178	pVCpu->hm.s.vmx.HCPhysVmcs));
4179	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4180	pCache->TestOut.HCPhysVmcs));
4181	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4182	pCache->TestOut.pCache));
4183	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4184	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4185	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4186	pCache->TestOut.pCtx));
4187	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4188	#endif
4189	return rc;
4190	}
4191
4192
4193	/**
4194	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4195	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4196	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4197	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4198	*
4199	* @returns VBox status code.
4200	* @param pVM Pointer to the VM.
4201	* @param pVCpu Pointer to the VMCPU.
4202	*/
4203	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4204	{
4205	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4206	{ \
4207	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4208	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4209	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4210	}
4211
4212	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4213	{ \
4214	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4215	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4216	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4217	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4218	}
4219
4220	AssertPtr(pVM);
4221	AssertPtr(pVCpu);
4222	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4223
4224	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4225	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4226	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4227	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4228	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4229	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4230	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4231	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4232	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4233
4234	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4235	#if 0
4236	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4237	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4238	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4239	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4240	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4241	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4242	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4243	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4244	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4245	#endif
4246
4247	/* 32-bit guest-state fields. */
4248	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4249	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4250	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4251	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4252	/* Unused 32-bit guest-state fields. */
4253	#if 0
4254	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4255	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4256	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE);
4257	#endif
4258
4259	/* Natural width guest-state fields. */
4260	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4261	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4262	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4263	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4264	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4265	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4267	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4268	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4275	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4276	/* Unused natural width guest-state fields. */
4277	#if 0
4278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4279	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4280	#endif
4281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4283
4284	if (pVM->hm.s.fNestedPaging)
4285	{
4286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4288	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4289	}
4290	else
4291	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4292
4293	#undef VMXLOCAL_INIT_VMCS_SELREG
4294	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4295	return VINF_SUCCESS;
4296	}
4297
4298
4299	/**
4300	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4301	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4302	* darwin, running 64-bit guests).
4303	*
4304	* @returns VBox status code.
4305	* @param pVCpu Pointer to the VMCPU.
4306	* @param idxField The VMCS field encoding.
4307	* @param u64Val 16, 32 or 64 bits value.
4308	*/
4309	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4310	{
4311	int rc;
4312	switch (idxField)
4313	{
4314	/*
4315	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4316	*/
4317	/* 64-bit Control fields. */
4318	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4319	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4320	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4321	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4322	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4323	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4324	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4325	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4326	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4327	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4328	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4329	case VMX_VMCS64_CTRL_EPTP_FULL:
4330	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4331	/* 64-bit Read-only data fields. */
4332	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4333	/* 64-bit Guest-state fields. */
4334	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4335	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4336	case VMX_VMCS64_GUEST_PAT_FULL:
4337	case VMX_VMCS64_GUEST_EFER_FULL:
4338	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4339	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4340	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4341	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4342	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4343	/* 64-bit Host-state fields. */
4344	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4345	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4346	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4347	{
4348	rc = VMXWriteVmcs32(idxField, u64Val);
4349	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4350	break;
4351	}
4352
4353	/*
4354	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4355	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4356	*/
4357	/* Natural-width Guest-state fields. */
4358	case VMX_VMCS_GUEST_CR0:
4359	case VMX_VMCS_GUEST_CR3:
4360	case VMX_VMCS_GUEST_CR4:
4361	case VMX_VMCS_GUEST_ES_BASE:
4362	case VMX_VMCS_GUEST_CS_BASE:
4363	case VMX_VMCS_GUEST_SS_BASE:
4364	case VMX_VMCS_GUEST_DS_BASE:
4365	case VMX_VMCS_GUEST_FS_BASE:
4366	case VMX_VMCS_GUEST_GS_BASE:
4367	case VMX_VMCS_GUEST_LDTR_BASE:
4368	case VMX_VMCS_GUEST_TR_BASE:
4369	case VMX_VMCS_GUEST_GDTR_BASE:
4370	case VMX_VMCS_GUEST_IDTR_BASE:
4371	case VMX_VMCS_GUEST_DR7:
4372	case VMX_VMCS_GUEST_RSP:
4373	case VMX_VMCS_GUEST_RIP:
4374	case VMX_VMCS_GUEST_RFLAGS:
4375	case VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS:
4376	case VMX_VMCS_GUEST_SYSENTER_ESP:
4377	case VMX_VMCS_GUEST_SYSENTER_EIP:
4378	{
4379	if ((u64Val >> 32ULL) == 0)
4380	{
4381	/* If this field is 64-bit, VT-x will zero out the top bits. */
4382	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4383	}
4384	else
4385	{
4386	/* Assert that only the 32->64 switcher case should ever come here. */
4387	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4388	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4389	}
4390	break;
4391	}
4392
4393	default:
4394	{
4395	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4396	rc = VERR_INVALID_PARAMETER;
4397	break;
4398	}
4399	}
4400	AssertRCReturn(rc, rc);
4401	return rc;
4402	}
4403
4404
4405	/**
4406	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4407	* hosts (except darwin) for 64-bit guests.
4408	*
4409	* @param pVCpu Pointer to the VMCPU.
4410	* @param idxField The VMCS field encoding.
4411	* @param u64Val 16, 32 or 64 bits value.
4412	*/
4413	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4414	{
4415	AssertPtr(pVCpu);
4416	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4417
4418	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4419	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4420
4421	/* Make sure there are no duplicates. */
4422	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
4423	{
4424	if (pCache->Write.aField[i] == idxField)
4425	{
4426	pCache->Write.aFieldVal[i] = u64Val;
4427	return VINF_SUCCESS;
4428	}
4429	}
4430
4431	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4432	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4433	pCache->Write.cValidEntries++;
4434	return VINF_SUCCESS;
4435	}
4436
4437
4438	/**
4439	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4440	*
4441	* @param pVCpu Pointer to the VMCPU.
4442	* @param pCache Pointer to the VMCS cache.
4443	*
4444	* @remarks No-long-jump zone!!!
4445	*/
4446	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4447	{
4448	AssertPtr(pCache);
4449	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4450	{
4451	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4452	AssertRC(rc);
4453	}
4454	pCache->Write.cValidEntries = 0;
4455	}
4456
4457
4458	/**
4459	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4460	*
4461	* @param pVCpu Pointer to the VMCPU.
4462	* @param pCache Pointer to the VMCS cache.
4463	*
4464	* @remarks No-long-jump zone!!!
4465	*/
4466	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4467	{
4468	AssertPtr(pCache);
4469	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4470	{
4471	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4472	AssertRC(rc);
4473	}
4474	}
4475	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4476
4477
4478	/**
4479	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4480	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4481	* timer.
4482	*
4483	* @returns VBox status code.
4484	* @param pVM Pointer to the VM.
4485	* @param pVCpu Pointer to the VMCPU.
4486	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4487	* out-of-sync. Make sure to update the required fields
4488	* before using them.
4489	* @remarks No-long-jump zone!!!
4490	*/
4491	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4492	{
4493	int rc = VERR_INTERNAL_ERROR_5;
4494	bool fOffsettedTsc = false;
4495	if (pVM->hm.s.vmx.fUsePreemptTimer)
4496	{
4497	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4498
4499	/* Make sure the returned values have sane upper and lower boundaries. */
4500	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4501	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4502	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4503	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4504
4505	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4506	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4507	}
4508	else
4509	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4510
4511	if (fOffsettedTsc)
4512	{
4513	uint64_t u64CurTSC = ASMReadTSC();
4514	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4515	{
4516	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4517	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4518
4519	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4520	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4521	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4522	}
4523	else
4524	{
4525	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4526	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4527	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4528	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4529	}
4530	}
4531	else
4532	{
4533	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4534	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4535	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4536	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4537	}
4538	}
4539
4540
4541	/**
4542	* Determines if an exception is a benign exception. Benign exceptions
4543	* are ones which cannot cause double-faults.
4544	*
4545	* @returns true if the exception is benign, false otherwise.
4546	* @param uVector The exception vector.
4547	*/
4548	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint32_t uVector)
4549	{
4550	switch (uVector)
4551	{
4552	case X86_XCPT_DB:
4553	case X86_XCPT_NMI:
4554	case X86_XCPT_BP:
4555	case X86_XCPT_OF:
4556	case X86_XCPT_BR:
4557	case X86_XCPT_UD:
4558	case X86_XCPT_NM:
4559	case X86_XCPT_CO_SEG_OVERRUN:
4560	case X86_XCPT_MF:
4561	case X86_XCPT_AC:
4562	case X86_XCPT_MC:
4563	case X86_XCPT_XF:
4564	return true;
4565	default:
4566	return false;
4567	}
4568	}
4569
4570
4571	/**
4572	* Determines if an exception is a contributory exception. Contributory
4573	* exceptions are ones which can cause double-faults.
4574	*
4575	* @returns true if the exception is contributory, false otherwise.
4576	* @param uVector The exception vector.
4577	*/
4578	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4579	{
4580	switch (uVector)
4581	{
4582	case X86_XCPT_GP:
4583	case X86_XCPT_SS:
4584	case X86_XCPT_NP:
4585	case X86_XCPT_TS:
4586	case X86_XCPT_DE:
4587	return true;
4588	default:
4589	return false;
4590	}
4591	return false;
4592	}
4593
4594
4595	/**
4596	* Determines if we are intercepting any contributory exceptions.
4597	*
4598	* @returns true if we are intercepting any contributory exception, false
4599	* otherwise.
4600	* @param pVCpu Pointer to the VMCPU.
4601	*/
4602	DECLINLINE(bool) hmR0VmxInterceptingContributoryXcpts(PVMCPU pVCpu)
4603	{
4604	if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_GP))
4605	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_SS))
4606	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_NP))
4607	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_TS))
4608	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_DE)))
4609	{
4610	return true;
4611	}
4612	return false;
4613	}
4614
4615
4616	/**
4617	* Handle a condition that occurred while delivering an event through the guest
4618	* IDT.
4619	*
4620	* @returns VBox status code (informational error codes included).
4621	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4622	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4623	* continue execution of the guest which will delivery the #DF.
4624	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4625	*
4626	* @param pVM Pointer to the VM.
4627	* @param pVCpu Pointer to the VMCPU.
4628	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4629	* out-of-sync. Make sure to update the required fields
4630	* before using them.
4631	* @param pVmxTransient Pointer to the VMX transient structure.
4632	*
4633	* @remarks No-long-jump zone!!!
4634	*/
4635	static int hmR0VmxCheckExitDueToEventDelivery(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4636	{
4637	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4638	AssertRCReturn(rc, rc);
4639	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4640	{
4641	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4642	AssertRCReturn(rc, rc);
4643
4644	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4645	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4646	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4647
4648	typedef enum
4649	{
4650	VMXREFLECTXCPT_XCPT, /* Reflect Idt-vectoring exception. */
4651	VMXREFLECTXCPT_DF, /* Reflect a double-fault to the guest. */
4652	VMXREFLECTXCPT_TF, /* Reflect a triple fault state to the VMM. */
4653	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4654	} VMXREFLECTXCPT;
4655
4656	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4657	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4658	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4659	{
4660	if ( hmR0VmxIsBenignXcpt(uIdtVector)
4661	\|\| hmR0VmxIsBenignXcpt(uExitVector)
4662	\|\| ( hmR0VmxIsContributoryXcpt(uIdtVector)
4663	&& uExitVector == X86_XCPT_PF))
4664	{
4665	enmReflect = VMXREFLECTXCPT_XCPT;
4666	}
4667	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF))
4668	&& uIdtVector == X86_XCPT_PF
4669	&& uExitVector == X86_XCPT_PF)
4670	{
4671	pVmxTransient->fVectoringPF = true;
4672	}
4673	else if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4674	&& hmR0VmxIsContributoryXcpt(uExitVector))
4675	{
4676	enmReflect = VMXREFLECTXCPT_DF;
4677	}
4678	else if ( hmR0VmxInterceptingContributoryXcpts(pVCpu)
4679	&& uIdtVector == X86_XCPT_PF
4680	&& hmR0VmxIsContributoryXcpt(uExitVector))
4681	{
4682	enmReflect = VMXREFLECTXCPT_DF;
4683	}
4684	else if (uIdtVector == X86_XCPT_DF)
4685	enmReflect = VMXREFLECTXCPT_TF;
4686	}
4687	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4688	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4689	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4690	{
4691	/*
4692	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4693	* (whatever they are) as they reoccur when restarting the instruction.
4694	*/
4695	enmReflect = VMXREFLECTXCPT_XCPT;
4696	}
4697
4698	Assert(pVmxTransient->fVectoringPF == false \|\| enmReflect == VMXREFLECTXCPT_NONE);
4699	switch (enmReflect)
4700	{
4701	case VMXREFLECTXCPT_XCPT:
4702	{
4703	Assert(!pVCpu->hm.s.Event.fPending);
4704	pVCpu->hm.s.Event.fPending = true;
4705	pVCpu->hm.s.Event.u64IntrInfo = VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo);
4706	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4707	{
4708	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4709	AssertRCReturn(rc, rc);
4710	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4711	}
4712	else
4713	pVCpu->hm.s.Event.u32ErrCode = 0;
4714	Log(("Pending event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4715	break;
4716	}
4717
4718	case VMXREFLECTXCPT_DF:
4719	{
4720	uint32_t u32IntrInfo;
4721	u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4722	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4723	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4724
4725	Assert(!pVCpu->hm.s.Event.fPending);
4726	pVCpu->hm.s.Event.fPending = true;
4727	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4728	pVCpu->hm.s.Event.u32ErrCode = 0;
4729	rc = VINF_VMX_DOUBLE_FAULT;
4730	Log(("Pending #DF %#RX64 uIdt=%#x uExit=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4731	break;
4732	}
4733
4734	case VMXREFLECTXCPT_TF:
4735	{
4736	Log(("Pending triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4737	rc = VINF_EM_RESET;
4738	break;
4739	}
4740
4741	default: /* shut up gcc. */
4742	break;
4743	}
4744	}
4745	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET \|\| rc == VINF_VMX_DOUBLE_FAULT);
4746	return rc;
4747	}
4748
4749
4750	/**
4751	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4752	*
4753	* @returns VBox status code.
4754	* @param pVM Pointer to the VM.
4755	* @param pVCpu Pointer to the VMCPU.
4756	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4757	* out-of-sync. Make sure to update the required fields
4758	* before using them.
4759	*
4760	* @remarks No-long-jump zone!!!
4761	*/
4762	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4763	{
4764	int rc = VINF_SUCCESS;
4765	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4766	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4767	{
4768	RTGCUINTREG uVal = 0;
4769	RTCCUINTREG uShadow = 0;
4770	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4771	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4772	AssertRCReturn(rc, rc);
4773	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4774	CPUMSetGuestCR0(pVCpu, uVal);
4775	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4776	}
4777	return rc;
4778	}
4779
4780
4781	/**
4782	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4783	*
4784	* @returns VBox status code.
4785	* @param pVM Pointer to the VM.
4786	* @param pVCpu Pointer to the VMCPU.
4787	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4788	* out-of-sync. Make sure to update the required fields
4789	* before using them.
4790	*
4791	* @remarks No-long-jump zone!!!
4792	*/
4793	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4794	{
4795	int rc = VINF_SUCCESS;
4796	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4797	{
4798	RTGCUINTREG uVal = 0;
4799	RTCCUINTREG uShadow = 0;
4800	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4801	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4802	AssertRCReturn(rc, rc);
4803	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4804	CPUMSetGuestCR4(pVCpu, uVal);
4805	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4806	}
4807	return rc;
4808	}
4809
4810
4811	/**
4812	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4813	*
4814	* @returns VBox status code.
4815	* @param pVM Pointer to the VM.
4816	* @param pVCpu Pointer to the VMCPU.
4817	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4818	* out-of-sync. Make sure to update the required fields
4819	* before using them.
4820	*
4821	* @remarks No-long-jump zone!!!
4822	*/
4823	DECLINLINE(int) hmR0VmxSaveGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4824	{
4825	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4826	return VINF_SUCCESS;
4827
4828	RTGCUINTREG uVal = 0;
4829	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4830	AssertRCReturn(rc, rc);
4831	pMixedCtx->rip = uVal;
4832	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4833	return rc;
4834	}
4835
4836
4837	/**
4838	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4839	*
4840	* @returns VBox status code.
4841	* @param pVM Pointer to the VM.
4842	* @param pVCpu Pointer to the VMCPU.
4843	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4844	* out-of-sync. Make sure to update the required fields
4845	* before using them.
4846	*
4847	* @remarks No-long-jump zone!!!
4848	*/
4849	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4850	{
4851	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4852	return VINF_SUCCESS;
4853
4854	RTGCUINTREG uVal = 0;
4855	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4856	AssertRCReturn(rc, rc);
4857	pMixedCtx->rsp = uVal;
4858	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4859	return rc;
4860	}
4861
4862
4863	/**
4864	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4865	*
4866	* @returns VBox status code.
4867	* @param pVM Pointer to the VM.
4868	* @param pVCpu Pointer to the VMCPU.
4869	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4870	* out-of-sync. Make sure to update the required fields
4871	* before using them.
4872	*
4873	* @remarks No-long-jump zone!!!
4874	*/
4875	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4876	{
4877	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4878	return VINF_SUCCESS;
4879
4880	RTGCUINTREG uVal = 0;
4881	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4882	AssertRCReturn(rc, rc);
4883	pMixedCtx->rflags.u64 = uVal;
4884
4885	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4886	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4887	{
4888	Assert(pVM->hm.s.vmx.pRealModeTSS);
4889	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4890	pMixedCtx->eflags.Bits.u1VM = 0;
4891	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4892	}
4893
4894	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4895	return rc;
4896	}
4897
4898
4899	/**
4900	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4901	* guest-CPU context.
4902	*/
4903	static int hmR0VmxSaveGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4904	{
4905	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4906	rc \|= hmR0VmxSaveGuestRsp(pVM, pVCpu, pMixedCtx);
4907	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
4908	return rc;
4909	}
4910
4911
4912	/**
4913	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4914	* from the guest-state area in the VMCS.
4915	*
4916	* @param pVM Pointer to the VM.
4917	* @param pVCpu Pointer to the VMCPU.
4918	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4919	* out-of-sync. Make sure to update the required fields
4920	* before using them.
4921	*
4922	* @remarks No-long-jump zone!!!
4923	*/
4924	DECLINLINE(void) hmR0VmxSaveGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4925	{
4926	uint32_t uIntrState = 0;
4927	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4928	AssertRC(rc);
4929
4930	if (!uIntrState)
4931	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4932	else
4933	{
4934	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4935	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4936	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4937	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* for hmR0VmxLoadGuestIntrState(). */
4938	AssertRC(rc);
4939	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4940	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4941	}
4942	}
4943
4944
4945	/**
4946	* Saves the guest's activity state.
4947	*
4948	* @returns VBox status code.
4949	* @param pVM Pointer to the VM.
4950	* @param pVCpu Pointer to the VMCPU.
4951	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4952	* out-of-sync. Make sure to update the required fields
4953	* before using them.
4954	*
4955	* @remarks No-long-jump zone!!!
4956	*/
4957	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4958	{
4959	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4960	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
4961	return VINF_SUCCESS;
4962	}
4963
4964
4965	/**
4966	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4967	* the current VMCS into the guest-CPU context.
4968	*
4969	* @returns VBox status code.
4970	* @param pVM Pointer to the VM.
4971	* @param pVCpu Pointer to the VMCPU.
4972	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4973	* out-of-sync. Make sure to update the required fields
4974	* before using them.
4975	*
4976	* @remarks No-long-jump zone!!!
4977	*/
4978	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4979	{
4980	int rc = VINF_SUCCESS;
4981	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4982	{
4983	uint32_t u32Val = 0;
4984	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4985	pMixedCtx->SysEnter.cs = u32Val;
4986	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4987	}
4988
4989	RTGCUINTREG uGCVal = 0;
4990	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4991	{
4992	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4993	pMixedCtx->SysEnter.eip = uGCVal;
4994	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4995	}
4996	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4997	{
4998	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4999	pMixedCtx->SysEnter.esp = uGCVal;
5000	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5001	}
5002	return rc;
5003	}
5004
5005
5006	/**
5007	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5008	* context.
5009	*
5010	* @returns VBox status code.
5011	* @param pVM Pointer to the VM.
5012	* @param pVCpu Pointer to the VMCPU.
5013	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5014	* out-of-sync. Make sure to update the required fields
5015	* before using them.
5016	*
5017	* @remarks No-long-jump zone!!!
5018	*/
5019	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5020	{
5021	RTGCUINTREG uVal = 0;
5022	int rc = VINF_SUCCESS;
5023	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
5024	{
5025	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
5026	pMixedCtx->fs.u64Base = uVal;
5027	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
5028	}
5029	return rc;
5030	}
5031
5032
5033	/**
5034	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5035	* context.
5036	*
5037	* @returns VBox status code.
5038	* @param pVM Pointer to the VM.
5039	* @param pVCpu Pointer to the VMCPU.
5040	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5041	* out-of-sync. Make sure to update the required fields
5042	* before using them.
5043	*
5044	* @remarks No-long-jump zone!!!
5045	*/
5046	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5047	{
5048	RTGCUINTREG uVal = 0;
5049	int rc = VINF_SUCCESS;
5050	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
5051	{
5052	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5053	pMixedCtx->gs.u64Base = uVal;
5054	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
5055	}
5056	return rc;
5057	}
5058
5059
5060	/**
5061	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5062	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5063	*
5064	* @returns VBox status code.
5065	* @param pVM Pointer to the VM.
5066	* @param pVCpu Pointer to the VMCPU.
5067	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5068	* out-of-sync. Make sure to update the required fields
5069	* before using them.
5070	*
5071	* @remarks No-long-jump zone!!!
5072	*/
5073	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5074	{
5075	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5076	return VINF_SUCCESS;
5077
5078	for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5079	{
5080	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5081	pMsr += i;
5082	switch (pMsr->u32IndexMSR)
5083	{
5084	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5085	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5086	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5087	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5088	#if 0
5089	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5090	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5091	#endif
5092	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5093	default:
5094	{
5095	AssertFailed();
5096	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5097	}
5098	}
5099	}
5100	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5101	return VINF_SUCCESS;
5102	}
5103
5104
5105	/**
5106	* Saves the guest control registers from the current VMCS into the guest-CPU
5107	* context.
5108	*
5109	* @returns VBox status code.
5110	* @param pVM Pointer to the VM.
5111	* @param pVCpu Pointer to the VMCPU.
5112	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5113	* out-of-sync. Make sure to update the required fields
5114	* before using them.
5115	*
5116	* @remarks No-long-jump zone!!!
5117	*/
5118	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5119	{
5120	RTGCUINTREG uVal = 0;
5121	RTGCUINTREG uShadow = 0;
5122	int rc = VINF_SUCCESS;
5123
5124	/* Guest CR0. Guest FPU. */
5125	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5126
5127	/* Guest CR4. */
5128	rc \|= hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
5129	AssertRCReturn(rc, rc);
5130
5131	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5132	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5133	{
5134	if ( pVM->hm.s.fNestedPaging
5135	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5136	{
5137	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5138	if (pMixedCtx->cr3 != uVal)
5139	{
5140	CPUMSetGuestCR3(pVCpu, uVal);
5141	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5142	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5143	}
5144
5145	/* We require EFER to check PAE mode. */
5146	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5147
5148	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5149	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5150	{
5151	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5152	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5153	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5154	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5155	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5156	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5157	}
5158	AssertRCReturn(rc, rc);
5159	}
5160	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5161	}
5162	return rc;
5163	}
5164
5165
5166	/**
5167	* Reads a guest segment register from the current VMCS into the guest-CPU
5168	* context.
5169	*
5170	* @returns VBox status code.
5171	* @param pVCpu Pointer to the VMCPU.
5172	* @param idxSel Index of the selector in the VMCS.
5173	* @param idxLimit Index of the segment limit in the VMCS.
5174	* @param idxBase Index of the segment base in the VMCS.
5175	* @param idxAccess Index of the access rights of the segment in the VMCS.
5176	* @param pSelReg Pointer to the segment selector.
5177	*
5178	* @remarks No-long-jump zone!!!
5179	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5180	* as that takes care of whether to read from the VMCS cache or not.
5181	*/
5182	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5183	PCPUMSELREG pSelReg)
5184	{
5185	uint32_t u32Val = 0;
5186	int rc = VMXReadVmcs32(idxSel, &u32Val);
5187	pSelReg->Sel = (uint16_t)u32Val;
5188	pSelReg->ValidSel = (uint16_t)u32Val;
5189	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5190
5191	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5192	pSelReg->u32Limit = u32Val;
5193
5194	RTGCUINTREG uGCVal = 0;
5195	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5196	pSelReg->u64Base = uGCVal;
5197
5198	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5199	pSelReg->Attr.u = u32Val;
5200	AssertRCReturn(rc, rc);
5201
5202	/*
5203	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5204	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5205	*/
5206	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5207	{
5208	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5209	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5210	}
5211	return rc;
5212	}
5213
5214
5215	/**
5216	* Saves the guest segment registers from the current VMCS into the guest-CPU
5217	* context.
5218	*
5219	* @returns VBox status code.
5220	* @param pVM Pointer to the VM.
5221	* @param pVCpu Pointer to the VMCPU.
5222	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5223	* out-of-sync. Make sure to update the required fields
5224	* before using them.
5225	*
5226	* @remarks No-long-jump zone!!!
5227	*/
5228	static int hmR0VmxSaveGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5229	{
5230	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5231	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5232	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5233	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5234	#else
5235	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5236	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5237	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5238	#endif
5239
5240	int rc = VINF_SUCCESS;
5241
5242	/* Guest segment registers. */
5243	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5244	{
5245	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5246	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5247	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5248	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5249	rc \|= VMXLOCAL_READ_SEG(ES, es);
5250	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5251	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5252	AssertRCReturn(rc, rc);
5253
5254	/* Restore segment attributes for real-on-v86 mode hack. */
5255	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5256	{
5257	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5258	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5259	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5260	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5261	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5262	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5263	}
5264	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5265	}
5266
5267	/* Guest LDTR. */
5268	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5269	{
5270	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5271	AssertRCReturn(rc, rc);
5272	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5273	}
5274
5275	/* Guest GDTR. */
5276	RTGCUINTREG uGCVal = 0;
5277	uint32_t u32Val = 0;
5278	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5279	{
5280	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5281	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5282	pMixedCtx->gdtr.pGdt = uGCVal;
5283	pMixedCtx->gdtr.cbGdt = u32Val;
5284	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5285	}
5286
5287	/* Guest IDTR. */
5288	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5289	{
5290	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5291	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5292	pMixedCtx->idtr.pIdt = uGCVal;
5293	pMixedCtx->idtr.cbIdt = u32Val;
5294	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5295	}
5296
5297	/* Guest TR. */
5298	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5299	{
5300	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5301
5302	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5303	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5304	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5305	AssertRCReturn(rc, rc);
5306	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5307	}
5308	return rc;
5309	}
5310
5311
5312	/**
5313	* Saves the guest debug registers from the current VMCS into the guest-CPU
5314	* context.
5315	*
5316	* @returns VBox status code.
5317	* @param pVM Pointer to the VM.
5318	* @param pVCpu Pointer to the VMCPU.
5319	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5320	* out-of-sync. Make sure to update the required fields
5321	* before using them.
5322	*
5323	* @remarks No-long-jump zone!!!
5324	*/
5325	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5326	{
5327	int rc = VINF_SUCCESS;
5328	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5329	{
5330	RTGCUINTREG uVal;
5331	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5332	pMixedCtx->dr[7] = uVal;
5333
5334	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5335	}
5336	return rc;
5337	}
5338
5339
5340	/**
5341	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5342	*
5343	* @returns VBox status code.
5344	* @param pVM Pointer to the VM.
5345	* @param pVCpu Pointer to the VMCPU.
5346	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5347	* out-of-sync. Make sure to update the required fields
5348	* before using them.
5349	*
5350	* @remarks No-long-jump zone!!!
5351	*/
5352	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5353	{
5354	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5355	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5356	return VINF_SUCCESS;
5357	}
5358
5359
5360	/**
5361	* Saves the entire guest state from the currently active VMCS into the
5362	* guest-CPU context. This essentially VMREADs all guest-data.
5363	*
5364	* @returns VBox status code.
5365	* @param pVM Pointer to the VM.
5366	* @param pVCpu Pointer to the VMCPU.
5367	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5368	* out-of-sync. Make sure to update the required fields
5369	* before using them.
5370	*/
5371	static int hmR0VmxSaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5372	{
5373	Assert(pVM);
5374	Assert(pVCpu);
5375	Assert(pMixedCtx);
5376
5377	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5378	return VINF_SUCCESS;
5379
5380	VMMRZCallRing3Disable(pVCpu);
5381
5382	int rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
5383	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGprs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5384
5385	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5386	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5387
5388	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5389	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5390
5391	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx);
5392	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5393
5394	rc = hmR0VmxSaveGuestSysenterMsrs(pVM, pVCpu, pMixedCtx);
5395	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5396
5397	rc = hmR0VmxSaveGuestFSBaseMsr(pVM, pVCpu, pMixedCtx);
5398	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5399
5400	rc = hmR0VmxSaveGuestGSBaseMsr(pVM, pVCpu, pMixedCtx);
5401	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5402
5403	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5404	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5405
5406	rc = hmR0VmxSaveGuestActivityState(pVM, pVCpu, pMixedCtx);
5407	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5408
5409	rc = hmR0VmxSaveGuestApicState(pVM, pVCpu, pMixedCtx);
5410	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5411
5412	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5413	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5414
5415	VMMRZCallRing3Enable(pVCpu);
5416	return rc;
5417	}
5418
5419
5420	/**
5421	* Check per-VM and per-VCPU force flag actions that require us to go back to
5422	* ring-3 for one reason or another.
5423	*
5424	* @returns VBox status code (information status code included).
5425	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5426	* ring-3.
5427	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5428	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5429	* interrupts)
5430	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5431	* all EMTs to be in ring-3.
5432	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5433	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5434	* to the EM loop.
5435	*
5436	* @param pVM Pointer to the VM.
5437	* @param pVCpu Pointer to the VMCPU.
5438	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5439	* out-of-sync. Make sure to update the required fields
5440	* before using them.
5441	*/
5442	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5443	{
5444	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5445
5446	int rc = VERR_INTERNAL_ERROR_5;
5447	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5448	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5449	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5450	{
5451	/* We need the control registers now, make sure the guest-CPU context is updated. */
5452	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5453	AssertRCReturn(rc, rc);
5454
5455	/* Pending HM CR3 sync. */
5456	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5457	{
5458	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5459	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5460	}
5461	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5462	{
5463	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5464	AssertRC(rc);
5465	}
5466
5467	/* Pending PGM C3 sync. */
5468	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5469	{
5470	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5471	if (rc != VINF_SUCCESS)
5472	{
5473	AssertRC(rc);
5474	Log2(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5475	return rc;
5476	}
5477	}
5478
5479	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5480	/* -XXX- what was that about single stepping? */
5481	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5482	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5483	{
5484	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5485	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5486	Log2(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5487	return rc;
5488	}
5489
5490	/* Pending VM request packets, such as hardware interrupts. */
5491	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5492	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5493	{
5494	Log2(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5495	return VINF_EM_PENDING_REQUEST;
5496	}
5497
5498	/* Pending PGM pool flushes. */
5499	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5500	{
5501	Log2(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5502	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5503	}
5504
5505	/* Pending DMA requests. */
5506	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5507	{
5508	Log2(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5509	return VINF_EM_RAW_TO_R3;
5510	}
5511	}
5512
5513	/* Paranoia. */
5514	Assert(rc != VERR_EM_INTERPRETER);
5515	return VINF_SUCCESS;
5516	}
5517
5518
5519	/**
5520	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5521	* VT-x to execute any instruction.
5522	*
5523	* @param pvCpu Pointer to the VMCPU.
5524	*/
5525	static void hmR0VmxUpdateTRPMTrap(PVMCPU pVCpu)
5526	{
5527	if (pVCpu->hm.s.Event.fPending)
5528	{
5529	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5530	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5531	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5532	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5533
5534	/* If a trap was already pending, we did something wrong! */
5535	Assert(TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP);
5536
5537	/* A page-fault exception during a page-fault would become a double-fault. */
5538	AssertMsg(uVectorType != VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT \|\| uVector != X86_XCPT_PF,
5539	("%#RX64 uVectorType=%#x uVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uVectorType, uVector));
5540
5541	TRPMEVENT enmTrapType;
5542	switch (uVectorType)
5543	{
5544	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5545	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5546	enmTrapType = TRPM_HARDWARE_INT;
5547	break;
5548	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5549	enmTrapType = TRPM_SOFTWARE_INT;
5550	break;
5551	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5552	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5553	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5554	enmTrapType = TRPM_TRAP;
5555	break;
5556	default:
5557	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5558	enmTrapType = TRPM_32BIT_HACK;
5559	break;
5560	}
5561	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5562	AssertRC(rc);
5563	if (fErrorCodeValid)
5564	TRPMSetErrorCode(pVCpu, uErrorCode);
5565
5566	/* Clear the VT-x state bits now that TRPM has the information. */
5567	pVCpu->hm.s.Event.fPending = false;
5568	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
5569	AssertRC(rc);
5570	}
5571	}
5572
5573
5574	/**
5575	* Does the necessary state syncing before doing a longjmp to ring-3.
5576	*
5577	* @param pVM Pointer to the VM.
5578	* @param pVCpu Pointer to the VMCPU.
5579	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5580	* out-of-sync. Make sure to update the required fields
5581	* before using them.
5582	* @param rcExit The reason for exiting to ring-3. Can be
5583	* VINF_VMM_UNKNOWN_RING3_CALL.
5584	*
5585	* @remarks No-long-jmp zone!!!
5586	*/
5587	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5588	{
5589	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5590
5591	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
5592	AssertRC(rc);
5593
5594	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5595	if (CPUMIsGuestFPUStateActive(pVCpu))
5596	{
5597	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5598	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5599	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5600	}
5601
5602	/* Restore debug registers if necessary and resync on next R0 reentry. */
5603	if (CPUMIsGuestDebugStateActive(pVCpu))
5604	{
5605	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5606	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5607	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5608	}
5609	else if (CPUMIsHyperDebugStateActive(pVCpu))
5610	{
5611	CPUMR0LoadHostDebugState(pVM, pVCpu);
5612	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5613	}
5614
5615	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5616	}
5617
5618
5619	/**
5620	* An action requires us to go back to ring-3. This function does the necessary
5621	* steps before we can safely return to ring-3. This is not the same as longjmps
5622	* to ring-3, this is voluntary.
5623	*
5624	* @param pVM Pointer to the VM.
5625	* @param pVCpu Pointer to the VMCPU.
5626	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5627	* out-of-sync. Make sure to update the required fields
5628	* before using them.
5629	* @param rcExit The reason for exiting to ring-3. Can be
5630	* VINF_VMM_UNKNOWN_RING3_CALL.
5631	*/
5632	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5633	{
5634	Assert(pVM);
5635	Assert(pVCpu);
5636	Assert(pMixedCtx);
5637	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5638
5639	/* We want to see what the guest-state was before VM-entry, don't resync here, as we will never continue guest execution.*/
5640	if (rcExit == VERR_VMX_INVALID_GUEST_STATE)
5641	return;
5642
5643	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5644	VMMRZCallRing3Disable(pVCpu);
5645	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5646
5647	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5648	hmR0VmxUpdateTRPMTrap(pVCpu);
5649
5650	/* Sync. the guest state. */
5651	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5652	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5653
5654	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5655	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5656	\| CPUM_CHANGED_LDTR
5657	\| CPUM_CHANGED_GDTR
5658	\| CPUM_CHANGED_IDTR
5659	\| CPUM_CHANGED_TR
5660	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5661
5662	/* On our way back from ring-3 the following needs to be done. */
5663	/** @todo This can change with preemption hooks. */
5664	if (rcExit == VINF_EM_RAW_INTERRUPT)
5665	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5666	else
5667	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5668
5669	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5670	VMMRZCallRing3Enable(pVCpu);
5671	}
5672
5673
5674	/**
5675	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5676	* longjump to ring-3 and possibly get preempted.
5677	*
5678	* @param pVCpu Pointer to the VMCPU.
5679	* @param enmOperation The operation causing the ring-3 longjump.
5680	* @param pvUser The user argument (pointer to the possibly
5681	* out-of-date guest-CPU context).
5682	*
5683	* @remarks Must never be called with @a enmOperation ==
5684	* VMMCALLRING3_VM_R0_ASSERTION.
5685	*/
5686	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5687	{
5688	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5689	Assert(pVCpu);
5690	Assert(pvUser);
5691	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5692	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5693
5694	VMMRZCallRing3Disable(pVCpu);
5695	Log(("hmR0VmxLongJmpToRing3\n"));
5696	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5697	VMMRZCallRing3Enable(pVCpu);
5698	}
5699
5700
5701	/**
5702	* Injects any pending TRPM trap into the VM by updating the VMCS.
5703	*
5704	* @returns VBox status code (informational status code included).
5705	* @param pVM Pointer to the VM.
5706	* @param pVCpu Pointer to the VMCPU.
5707	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5708	* out-of-sync. Make sure to update the required fields
5709	* before using them.
5710	*/
5711	static int hmR0VmxInjectTRPMTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5712	{
5713	if (!TRPMHasTrap(pVCpu))
5714	return VINF_SUCCESS;
5715
5716	uint8_t u8Vector = 0;
5717	TRPMEVENT enmTrpmEvent = TRPM_SOFTWARE_INT;
5718	RTGCUINT uErrCode = 0;
5719
5720	int rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmTrpmEvent, &uErrCode, NULL /* puCr2 */);
5721	AssertRCReturn(rc, rc);
5722	Assert(enmTrpmEvent != TRPM_SOFTWARE_INT);
5723
5724	rc = TRPMResetTrap(pVCpu);
5725	AssertRCReturn(rc, rc);
5726
5727	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5728	uint32_t u32IntrInfo = u8Vector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5729	if (enmTrpmEvent == TRPM_TRAP)
5730	{
5731	switch (u8Vector)
5732	{
5733	case X86_XCPT_BP:
5734	case X86_XCPT_OF:
5735	{
5736	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5737	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5738	break;
5739	}
5740
5741	case X86_XCPT_DF:
5742	case X86_XCPT_TS:
5743	case X86_XCPT_NP:
5744	case X86_XCPT_SS:
5745	case X86_XCPT_GP:
5746	case X86_XCPT_PF:
5747	case X86_XCPT_AC:
5748	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5749	them which VT-x/VMM pushes to the guest stack. */
5750	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5751	/* no break! */
5752	default:
5753	{
5754	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5755	break;
5756	}
5757	}
5758	}
5759	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5760	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5761	else
5762	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5763
5764	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5765	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, uErrCode);
5766	}
5767
5768
5769	/**
5770	* Checks if there are any pending guest interrupts to be delivered and injects
5771	* them into the VM by updating the VMCS.
5772	*
5773	* @returns VBox status code (informational status codes included).
5774	* @param pVM Pointer to the VM.
5775	* @param pVCpu Pointer to the VMCPU.
5776	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5777	* out-of-sync. Make sure to update the required fields
5778	* before using them.
5779	*
5780	* @remarks Must be called after hmR0VmxLoadGuestIntrState().
5781	*/
5782	static int hmR0VmxInjectPendingInterrupt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5783	{
5784	/* First inject any pending HM interrupts. */
5785	if (pVCpu->hm.s.Event.fPending)
5786	{
5787	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, 0 /* cbInstr */,
5788	pVCpu->hm.s.Event.u32ErrCode);
5789	AssertRCReturn(rc, rc);
5790	pVCpu->hm.s.Event.fPending = false;
5791	return rc;
5792	}
5793
5794	/** @todo SMI. SMIs take priority over NMIs. */
5795
5796	/* NMI. NMIs take priority over regular interrupts . */
5797	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
5798	{
5799	/* Construct an NMI interrupt and inject it into the VMCS. */
5800	RTGCUINTPTR uIntrInfo;
5801	uIntrInfo = X86_XCPT_NMI;
5802	uIntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5803	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5804	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5805	AssertRCReturn(rc, rc);
5806	return rc;
5807	}
5808
5809	/* We need the guests's RFLAGS for sure from this point on, make sure it is updated. */
5810	int rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
5811	AssertRCReturn(rc, rc);
5812
5813	/* If there isn't any active trap, check if we have pending interrupts and convert them to TRPM traps and deliver them. */
5814	if (!TRPMHasTrap(pVCpu))
5815	{
5816	/* Check if there are guest external interrupts (PIC/APIC) pending. */
5817	if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5818	{
5819	/*
5820	* If the guest can receive interrupts now (interrupts enabled and no interrupt inhibition is active) convert
5821	* the PDM interrupt into a TRPM event and inject it.
5822	*/
5823	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5824	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5825	{
5826	uint8_t u8Interrupt = 0;
5827	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5828	if (RT_SUCCESS(rc))
5829	{
5830	/* Convert pending interrupt from PIC/APIC into TRPM and handle it below. */
5831	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
5832	AssertRCReturn(rc, rc);
5833	}
5834	else
5835	{
5836	/** @todo Does this actually happen? If not turn it into an assertion. */
5837	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5838	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5839	}
5840	}
5841	else if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5842	{
5843	/* Instruct VT-x to cause an interrupt-window exit as soon as the guest is ready to receive interrupts again. */
5844	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5845	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5846	AssertRCReturn(rc, rc);
5847	}
5848	/* else we will deliver interrupts whenever the guest exits next and it's in a state to receive interrupts. */
5849	}
5850	}
5851
5852	/* If interrupts can be delivered, inject it into the VM. */
5853	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5854	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
5855	&& TRPMHasTrap(pVCpu))
5856	{
5857	rc = hmR0VmxInjectTRPMTrap(pVM, pVCpu, pMixedCtx);
5858	AssertRCReturn(rc, rc);
5859	}
5860	return rc;
5861	}
5862
5863	/**
5864	* Injects an invalid-opcode (#UD) exception into the VM.
5865	*
5866	* @returns VBox status code (informational status code included).
5867	* @param pVM Pointer to the VM.
5868	* @param pVCpu Pointer to the VMCPU.
5869	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5870	* out-of-sync. Make sure to update the required fields
5871	* before using them.
5872	*/
5873	DECLINLINE(int) hmR0VmxInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5874	{
5875	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5876	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5877	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5878	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5879	}
5880
5881
5882	/**
5883	* Injects a double-fault (#DF) exception into the VM.
5884	*
5885	* @returns VBox status code (informational status code included).
5886	* @param pVM Pointer to the VM.
5887	* @param pVCpu Pointer to the VMCPU.
5888	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5889	* out-of-sync. Make sure to update the required fields
5890	* before using them.
5891	*/
5892	DECLINLINE(int) hmR0VmxInjectXcptDF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5893	{
5894	/* Inject the double-fault. */
5895	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5896	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5897	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5898	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5899	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5900	}
5901
5902
5903	/**
5904	* Injects a debug (#DB) exception into the VM.
5905	*
5906	* @returns VBox status code (informational status code included).
5907	* @param pVM Pointer to the VM.
5908	* @param pVCpu Pointer to the VMCPU.
5909	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5910	* out-of-sync. Make sure to update the required fields
5911	* before using them.
5912	*/
5913	DECLINLINE(int) hmR0VmxInjectXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5914	{
5915	/* Inject the debug-exception. */
5916	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5917	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5918	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5919	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5920	}
5921
5922
5923	/**
5924	* Injects a overflow (#OF) exception into the VM.
5925	*
5926	* @returns VBox status code (informational status code included).
5927	* @param pVM Pointer to the VM.
5928	* @param pVCpu Pointer to the VMCPU.
5929	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5930	* out-of-sync. Make sure to update the required fields
5931	* before using them.
5932	* @param cbInstr The value of RIP that is to be pushed on the guest
5933	* stack.
5934	*/
5935	DECLINLINE(int) hmR0VmxInjectXcptOF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5936	{
5937	/* Inject the overflow exception. */
5938	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5939	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5940	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5941	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5942	}
5943
5944
5945	/**
5946	* Injects a general-protection (#GP) fault into the VM.
5947	*
5948	* @returns VBox status code (informational status code included).
5949	* @param pVM Pointer to the VM.
5950	* @param pVCpu Pointer to the VMCPU.
5951	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5952	* out-of-sync. Make sure to update the required fields
5953	* before using them.
5954	* @param u32ErrorCode The error code associated with the #GP.
5955	*/
5956	DECLINLINE(int) hmR0VmxInjectXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode)
5957	{
5958	/* Inject the general-protection fault. */
5959	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5960	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5961	if (fErrorCodeValid)
5962	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5963	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5964	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode);
5965	}
5966
5967
5968	/**
5969	* Injects a software interrupt (INTn) into the VM.
5970	*
5971	* @returns VBox status code (informational status code included).
5972	* @param pVM Pointer to the VM.
5973	* @param pVCpu Pointer to the VMCPU.
5974	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5975	* out-of-sync. Make sure to update the required fields
5976	* before using them.
5977	* @param uVector The software interrupt vector number.
5978	* @param cbInstr The value of RIP that is to be pushed on the guest
5979	* stack.
5980	*/
5981	DECLINLINE(int) hmR0VmxInjectIntN(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
5982	{
5983	/* Inject the INTn. */
5984	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5985	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5986	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5987	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5988	}
5989
5990
5991	/**
5992	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
5993	* stack.
5994	*
5995	* @returns VBox status code (information status code included).
5996	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
5997	* @param pVM Pointer to the VM.
5998	* @param pMixedCtx Pointer to the guest-CPU context.
5999	* @param uValue The value to push to the guest stack.
6000	*/
6001	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6002	{
6003	/*
6004	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6005	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6006	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6007	*/
6008	if (pMixedCtx->sp == 1)
6009	return VINF_EM_RESET;
6010	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6011	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6012	AssertRCReturn(rc, rc);
6013	return rc;
6014	}
6015
6016
6017	/**
6018	* Injects an event into the guest upon VM-entry by updating the relevant fields
6019	* in the VM-entry area in the VMCS.
6020	*
6021	* @returns VBox status code (informational error codes included).
6022	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6023	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6024	*
6025	* @param pVM Pointer to the VM.
6026	* @param pVCpu Pointer to the VMCPU.
6027	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6028	* out-of-sync. Make sure to update the required fields
6029	* before using them.
6030	* @param u64IntrInfo The VM-entry interruption-information field.
6031	* @param cbInstr The VM-entry instruction length in bytes (for software
6032	* interrupts, exceptions and privileged software
6033	* exceptions).
6034	* @param u32ErrCode The VM-entry exception error code.
6035	*
6036	* @remarks No-long-jump zone!!!
6037	*/
6038	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6039	uint32_t u32ErrCode)
6040	{
6041	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6042	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6043	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6044
6045	/* We require CR0 to check if the guest is in real-mode. */
6046	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6047	AssertRCReturn(rc, rc);
6048
6049	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6050	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6051
6052	/*
6053	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6054	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6055	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6056	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6057	*/
6058	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6059	{
6060	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6061	{
6062	Assert(PDMVmmDevHeapIsEnabled(pVM));
6063	Assert(pVM->hm.s.vmx.pRealModeTSS);
6064
6065	/* Save the required guest state bits from the VMCS. */
6066	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
6067	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
6068	AssertRCReturn(rc, rc);
6069
6070	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6071	const size_t cbIdtEntry = 4;
6072	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6073	{
6074	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6075	if (uVector == X86_XCPT_DF)
6076	return VINF_EM_RESET;
6077	else if (uVector == X86_XCPT_GP)
6078	{
6079	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6080	return hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
6081	}
6082
6083	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6084	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6085	return hmR0VmxInjectXcptGP(pVM, pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */);
6086	}
6087
6088	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6089	uint16_t uGuestIp = pMixedCtx->ip;
6090	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6091	{
6092	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6093	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6094	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6095	}
6096	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6097	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6098
6099	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6100	uint16_t offIdtEntry = 0;
6101	RTSEL selIdtEntry = 0;
6102	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6103	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6104	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6105	AssertRCReturn(rc, rc);
6106
6107	/* Construct the stack frame for the interrupt/exception handler. */
6108	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6109	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6110	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6111	AssertRCReturn(rc, rc);
6112
6113	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6114	if (rc == VINF_SUCCESS)
6115	{
6116	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6117	pMixedCtx->rip = offIdtEntry;
6118	pMixedCtx->cs.Sel = selIdtEntry;
6119	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6120	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6121	\| HM_CHANGED_GUEST_RIP
6122	\| HM_CHANGED_GUEST_RFLAGS
6123	\| HM_CHANGED_GUEST_RSP;
6124	}
6125	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6126	return rc;
6127	}
6128	else
6129	{
6130	/*
6131	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6132	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6133	*/
6134	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6135	}
6136	}
6137
6138	/* Add the valid bit, maybe the caller was lazy. */
6139	u32IntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6140
6141	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6142	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6143	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6144
6145	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6146	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6147	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6148	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6149	AssertRCReturn(rc, rc);
6150	return rc;
6151	}
6152
6153
6154	/**
6155	* Enters the VT-x session.
6156	*
6157	* @returns VBox status code.
6158	* @param pVM Pointer to the VM.
6159	* @param pVCpu Pointer to the VMCPU.
6160	* @param pCpu Pointer to the CPU info struct.
6161	*/
6162	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6163	{
6164	AssertPtr(pVM);
6165	AssertPtr(pVCpu);
6166	Assert(pVM->hm.s.vmx.fSupported);
6167	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6168	NOREF(pCpu);
6169
6170	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6171
6172	/* Make sure we're in VMX root mode. */
6173	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6174	if (!(u32HostCR4 & X86_CR4_VMXE))
6175	{
6176	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6177	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6178	}
6179
6180	/* Load the active VMCS as the current one. */
6181	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6182	if (RT_FAILURE(rc))
6183	return rc;
6184
6185	/** @todo this will change with preemption hooks where can can VMRESUME as long
6186	* as we're no preempted. */
6187	pVCpu->hm.s.fResumeVM = false;
6188	return VINF_SUCCESS;
6189	}
6190
6191
6192	/**
6193	* Leaves the VT-x session.
6194	*
6195	* @returns VBox status code.
6196	* @param pVM Pointer to the VM.
6197	* @param pVCpu Pointer to the VMCPU.
6198	* @param pCtx Pointer to the guest-CPU context.
6199	*/
6200	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6201	{
6202	AssertPtr(pVCpu);
6203	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6204	NOREF(pVM);
6205	NOREF(pCtx);
6206
6207	/** @todo this will change with preemption hooks where we only VMCLEAR when
6208	* we are actually going to be preempted, not all the time like we
6209	* currently do. */
6210	/*
6211	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6212	* and mark the VMCS launch-state as "clear".
6213	*/
6214	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6215	return rc;
6216	}
6217
6218
6219	/**
6220	* Saves the host state in the VMCS host-state.
6221	* Sets up the VM-exit MSR-load area.
6222	*
6223	* The CPU state will be loaded from these fields on every successful VM-exit.
6224	*
6225	* @returns VBox status code.
6226	* @param pVM Pointer to the VM.
6227	* @param pVCpu Pointer to the VMCPU.
6228	*
6229	* @remarks No-long-jump zone!!!
6230	*/
6231	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6232	{
6233	AssertPtr(pVM);
6234	AssertPtr(pVCpu);
6235	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6236
6237	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6238
6239	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6240	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6241	return VINF_SUCCESS;
6242
6243	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6244	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6245
6246	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6247	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6248
6249	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6250	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6251
6252	/* Reset the host-state-changed flag. */
6253	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6254	return rc;
6255	}
6256
6257
6258	/**
6259	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6260	* loaded from these fields on every successful VM-entry.
6261	*
6262	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6263	* Sets up the VM-entry controls.
6264	* Sets up the appropriate VMX non-root function to execute guest code based on
6265	* the guest CPU mode.
6266	*
6267	* @returns VBox status code.
6268	* @param pVM Pointer to the VM.
6269	* @param pVCpu Pointer to the VMCPU.
6270	* @param pCtx Pointer to the guest-CPU context.
6271	*
6272	* @remarks No-long-jump zone!!!
6273	*/
6274	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6275	{
6276	AssertPtr(pVM);
6277	AssertPtr(pVCpu);
6278	AssertPtr(pCtx);
6279	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6280
6281	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6282
6283	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6284
6285	/* Determine real-on-v86 mode. */
6286	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6287	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6288	&& CPUMIsGuestInRealModeEx(pCtx))
6289	{
6290	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6291	}
6292
6293	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6294	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6295
6296	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu, pCtx);
6297	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6298
6299	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6300	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6301
6302	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6303	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6304
6305	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6306	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6307
6308	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6309	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6310
6311	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6312	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6313
6314	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6315	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6316
6317	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6318	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6319
6320	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6321	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6322
6323	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6324	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6325	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6326
6327	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6328	return rc;
6329	}
6330
6331
6332	/**
6333	* Does the preparations before executing guest code in VT-x.
6334	*
6335	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6336	* recompiler. We must be cautious what we do here regarding committing
6337	* guest-state information into the the VMCS assuming we assuredly execute the
6338	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6339	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6340	* that the recompiler can (and should) use them when it resumes guest
6341	* execution. Otherwise such operations must be done when we can no longer
6342	* exit to ring-3.
6343	*
6344	* @returns VBox status code (informational status codes included).
6345	* @retval VINF_SUCCESS if we can proceed with running the guest.
6346	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6347	* into the guest.
6348	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6349	*
6350	* @param pVM Pointer to the VM.
6351	* @param pVCpu Pointer to the VMCPU.
6352	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6353	* out-of-sync. Make sure to update the required fields
6354	* before using them.
6355	* @param pVmxTransient Pointer to the VMX transient structure.
6356	*
6357	* @remarks Called with preemption disabled.
6358	*/
6359	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6360	{
6361	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6362
6363	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6364	PGMRZDynMapFlushAutoSet(pVCpu);
6365	#endif
6366
6367	/* Check force flag actions that might require us to go back to ring-3. */
6368	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6369	if (rc != VINF_SUCCESS)
6370	return rc;
6371
6372	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6373	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6374	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6375	{
6376	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6377	RTGCPHYS GCPhysApicBase;
6378	GCPhysApicBase = pMixedCtx->msrApicBase;
6379	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6380
6381	/* Unalias any existing mapping. */
6382	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6383	AssertRCReturn(rc, rc);
6384
6385	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6386	Log2(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6387	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6388	AssertRCReturn(rc, rc);
6389
6390	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6391	}
6392
6393	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6394	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6395	pVmxTransient->uEFlags = ASMIntDisableFlags();
6396	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6397	{
6398	ASMSetFlags(pVmxTransient->uEFlags);
6399	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6400	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6401	return VINF_EM_RAW_INTERRUPT;
6402	}
6403	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6404	#endif
6405
6406	/*
6407	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6408	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6409	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6410	*/
6411	hmR0VmxLoadGuestIntrState(pVM, pVCpu, pMixedCtx);
6412	rc = hmR0VmxInjectPendingInterrupt(pVM, pVCpu, pMixedCtx);
6413	AssertRCReturn(rc, rc);
6414	return rc;
6415	}
6416
6417
6418	/**
6419	* Prepares to run guest code in VT-x and we've committed to doing so. This
6420	* means there is no backing out to ring-3 or anywhere else at this
6421	* point.
6422	*
6423	* @param pVM Pointer to the VM.
6424	* @param pVCpu Pointer to the VMCPU.
6425	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6426	* out-of-sync. Make sure to update the required fields
6427	* before using them.
6428	* @param pVmxTransient Pointer to the VMX transient structure.
6429	*
6430	* @remarks Called with preemption disabled.
6431	* @remarks No-long-jump zone!!!
6432	*/
6433	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6434	{
6435	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6436	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6437
6438	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6439	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6440	pVmxTransient->uEFlags = ASMIntDisableFlags();
6441	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6442	#endif
6443
6444	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6445	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6446	Log(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6447	int rc = VINF_SUCCESS;
6448	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6449	{
6450	rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pMixedCtx);
6451	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6452	}
6453	else if (pVCpu->hm.s.fContextUseFlags)
6454	{
6455	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6456	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6457	}
6458	AssertRC(rc);
6459	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6460
6461	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6462	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6463	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6464
6465	Assert(pVM->hm.s.vmx.pfnFlushTaggedTlb);
6466	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6467	pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu); /* Flush the TLB of guest entries as necessary. */
6468
6469	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6470	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6471	{
6472	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6473	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6474	}
6475
6476	/*
6477	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6478	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6479	*/
6480	if (pVM->hm.s.fTPRPatchingActive)
6481	{
6482	Assert(!CPUMIsGuestInLongMode(pVCpu));
6483
6484	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6485	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6486	AssertRC(rc);
6487
6488	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6489	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6490	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6491	see hmR0VmxLoadGuestApicState(). */
6492	}
6493
6494	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6495	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6496	to start executing. */
6497	}
6498
6499
6500	/**
6501	* Performs some essential restoration of state after running guest code in
6502	* VT-x.
6503	*
6504	* @param pVM Pointer to the VM.
6505	* @param pVCpu Pointer to the VMCPU.
6506	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6507	* out-of-sync. Make sure to update the required fields
6508	* before using them.
6509	* @param pVmxTransient Pointer to the VMX transient structure.
6510	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6511	*
6512	* @remarks Called with interrupts disabled.
6513	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6514	* unconditionally when it is safe to do so.
6515	*/
6516	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6517	{
6518	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6519	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6520
6521	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6522	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6523	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6524	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6525	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6526
6527	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6528	{
6529	/** @todo Find a way to fix hardcoding a guestimate. */
6530	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6531	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6532	}
6533
6534	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6535	Assert(!(ASMGetFlags() & X86_EFL_IF));
6536	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6537
6538	/* Restore the effects of TPR patching if any. */
6539	if (pVM->hm.s.fTPRPatchingActive)
6540	{
6541	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6542	AssertRC(rc);
6543	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6544	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6545	}
6546
6547	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6548	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6549
6550	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6551	uint32_t uExitReason;
6552	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6553	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6554	AssertRC(rc);
6555	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6556	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6557
6558	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6559	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6560
6561	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6562	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6563	{
6564	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6565	return;
6566	}
6567
6568	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6569	{
6570	/* Update the guest interruptibility-state from the VMCS. */
6571	hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
6572
6573	/*
6574	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6575	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6576	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6577	*/
6578	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6579	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6580	{
6581	Log(("Setting TPR=%d\n", pVCpu->hm.s.vmx.pbVirtApic[0x80]));
6582	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6583	AssertRC(rc);
6584	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6585	}
6586	}
6587	}
6588
6589
6590	/**
6591	* Runs the guest code using VT-x.
6592	*
6593	* @returns VBox status code.
6594	* @param pVM Pointer to the VM.
6595	* @param pVCpu Pointer to the VMCPU.
6596	* @param pCtx Pointer to the guest-CPU context.
6597	*
6598	* @remarks Called with preemption disabled.
6599	*/
6600	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6601	{
6602	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6603	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6604
6605	VMXTRANSIENT VmxTransient;
6606	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6607	int rc = VERR_INTERNAL_ERROR_5;
6608	unsigned cLoops = 0;
6609
6610	for (;; cLoops++)
6611	{
6612	Assert(!HMR0SuspendPending());
6613	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6614	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6615	(unsigned)RTMpCpuId(), cLoops));
6616
6617	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6618	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6619	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6620	if (rc != VINF_SUCCESS)
6621	break;
6622
6623	/*
6624	* No longjmps to ring-3 from this point on!!!
6625	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6626	* This also disables flushing of the R0-logger instance (if any).
6627	*/
6628	VMMRZCallRing3Disable(pVCpu);
6629	VMMRZCallRing3RemoveNotification(pVCpu);
6630	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6631
6632	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6633	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6634
6635	/*
6636	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6637	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6638	*/
6639	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6640	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6641	{
6642	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6643	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6644	return rc;
6645	}
6646
6647	/* Handle the VM-exit. */
6648	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6649	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6650	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6651	rc = (*s_apfnVMExitHandlers[VmxTransient.uExitReason])(pVM, pVCpu, pCtx, &VmxTransient);
6652	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6653	if (rc != VINF_SUCCESS)
6654	break;
6655	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6656	{
6657	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6658	rc = VINF_EM_RAW_INTERRUPT;
6659	break;
6660	}
6661	}
6662
6663	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6664	if (rc == VERR_EM_INTERPRETER)
6665	rc = VINF_EM_RAW_EMULATE_INSTR;
6666	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6667	return rc;
6668	}
6669
6670	#if 0
6671	DECLINLINE(int) hmR0VmxHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, unsigned rcReason)
6672	{
6673	int rc;
6674	switch (rcReason)
6675	{
6676	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6677	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6678	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6679	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6680	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6681	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6682	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6683	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6684	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6685	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6686	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6687	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6688	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6689	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6690	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6691	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6692	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6693	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6694	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6695	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6696	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6697	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6698	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6699	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6700	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6701	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6702	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6703	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6704	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6705	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6706	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6707	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6708	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6709
6710	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6711	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6712	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6713	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6714	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6715	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6716	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6717	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6718	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6719
6720	case VMX_EXIT_VMCALL:
6721	case VMX_EXIT_VMCLEAR:
6722	case VMX_EXIT_VMLAUNCH:
6723	case VMX_EXIT_VMPTRLD:
6724	case VMX_EXIT_VMPTRST:
6725	case VMX_EXIT_VMREAD:
6726	case VMX_EXIT_VMRESUME:
6727	case VMX_EXIT_VMWRITE:
6728	case VMX_EXIT_VMXOFF:
6729	case VMX_EXIT_VMXON:
6730	case VMX_EXIT_INVEPT:
6731	case VMX_EXIT_INVVPID:
6732	case VMX_EXIT_VMFUNC:
6733	rc = hmR0VmxExitInjectXcptUD(pVM, pVCpu, pMixedCtx, pVmxTransient);
6734	break;
6735	default:
6736	rc = hmR0VmxExitErrUndefined(pVM, pVCpu, pMixedCtx, pVmxTransient);
6737	break;
6738	}
6739	return rc;
6740	}
6741	#endif
6742
6743	#ifdef DEBUG
6744	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6745	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6746	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6747	# define VMX_ASSERT_PREEMPT_CPUID() \
6748	do \
6749	{ \
6750	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6751	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6752	} while (0)
6753
6754	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6755	do { \
6756	AssertPtr(pVM); \
6757	AssertPtr(pVCpu); \
6758	AssertPtr(pMixedCtx); \
6759	AssertPtr(pVmxTransient); \
6760	Assert(pVmxTransient->fVMEntryFailed == false); \
6761	Assert(ASMIntAreEnabled()); \
6762	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6763	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6764	LogFunc(("\n")); \
6765	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6766	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6767	VMX_ASSERT_PREEMPT_CPUID(); \
6768	} while (0)
6769	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6770	do { \
6771	LogFunc(("\n")); \
6772	} while(0)
6773	#else /* Release builds */
6774	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6775	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6776	#endif
6777
6778
6779	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6780	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6781	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6782	/**
6783	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6784	*/
6785	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6786	{
6787	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6788	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6789	return VINF_SUCCESS;
6790	}
6791
6792
6793	/**
6794	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6795	*/
6796	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6797	{
6798	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6799	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6800	AssertRCReturn(rc, rc);
6801
6802	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6803	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6804	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6805
6806	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6807	return VINF_EM_RAW_INTERRUPT;
6808
6809	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6810	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
6811	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6812	return VINF_SUCCESS;
6813	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6814	return rc;
6815
6816	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6817	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6818	switch (uIntrType)
6819	{
6820	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6821	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6822	/* no break */
6823	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
6824	{
6825	switch (uVector)
6826	{
6827	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6828	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6829	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6830	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6831	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6832	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6833	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
6834	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6835	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6836	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6837	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6838	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
6839	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6840	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
6841	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6842	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
6843	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6844	#endif
6845	default:
6846	{
6847	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6848	AssertRCReturn(rc, rc);
6849
6850	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
6851	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6852	{
6853	Assert(pVM->hm.s.vmx.pRealModeTSS);
6854	Assert(PDMVmmDevHeapIsEnabled(pVM));
6855	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6856	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
6857	AssertRCReturn(rc, rc);
6858	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
6859	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
6860	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
6861	AssertRCReturn(rc, rc);
6862	}
6863	else
6864	{
6865	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
6866	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
6867	}
6868	break;
6869	}
6870	}
6871	break;
6872	}
6873
6874	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
6875	default:
6876	{
6877	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
6878	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
6879	break;
6880	}
6881	}
6882	return rc;
6883	}
6884
6885
6886	/**
6887	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
6888	*/
6889	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6890	{
6891	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6892
6893	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
6894	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
6895	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
6896	AssertRCReturn(rc, rc);
6897
6898	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and resume guest execution. */
6899	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6900	return VINF_SUCCESS;
6901	}
6902
6903
6904	/**
6905	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
6906	*/
6907	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6908	{
6909	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6910	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
6911	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6912	}
6913
6914
6915	/**
6916	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
6917	*/
6918	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6919	{
6920	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6921	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6922	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6923	AssertRCReturn(rc, rc);
6924
6925	pMixedCtx->rip += pVmxTransient->cbInstr;
6926	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6927
6928	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6929	return VINF_SUCCESS;
6930	}
6931
6932
6933	/**
6934	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
6935	*/
6936	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6937	{
6938	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6939	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6940	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6941	AssertRCReturn(rc, rc);
6942
6943	pMixedCtx->rip += pVmxTransient->cbInstr;
6944	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6945
6946	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6947	return VINF_SUCCESS;
6948	}
6949
6950
6951	/**
6952	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
6953	*/
6954	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6955	{
6956	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6957	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6958	if (RT_LIKELY(rc == VINF_SUCCESS))
6959	{
6960	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6961	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6962	AssertRCReturn(rc, rc);
6963	Assert(pVmxTransient->cbInstr == 2);
6964
6965	Log(("hmR0VmxExitCpuid: RIP=%#RX64\n", pMixedCtx->rip));
6966	pMixedCtx->rip += pVmxTransient->cbInstr;
6967	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6968	}
6969	else
6970	{
6971	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6972	rc = VERR_EM_INTERPRETER;
6973	}
6974	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6975	return rc;
6976	}
6977
6978
6979	/**
6980	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
6981	*/
6982	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6983	{
6984	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6985	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
6986	AssertRCReturn(rc, rc);
6987
6988	if (pMixedCtx->cr4 & X86_CR4_SMXE)
6989	return VINF_EM_RAW_EMULATE_INSTR;
6990
6991	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
6992	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6993	}
6994
6995
6996	/**
6997	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
6998	*/
6999	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7000	{
7001	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7002	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7003	AssertRCReturn(rc, rc);
7004
7005	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7006	if (RT_LIKELY(rc == VINF_SUCCESS))
7007	{
7008	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7009	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7010	AssertRCReturn(rc, rc);
7011	Assert(pVmxTransient->cbInstr == 2);
7012
7013	pMixedCtx->rip += pVmxTransient->cbInstr;
7014	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7015
7016	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7017	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7018	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7019	}
7020	else
7021	{
7022	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7023	rc = VERR_EM_INTERPRETER;
7024	}
7025	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7026	return rc;
7027	}
7028
7029
7030	/**
7031	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7032	*/
7033	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7034	{
7035	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7036	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7037	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7038	AssertRCReturn(rc, rc);
7039
7040	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7041	if (RT_LIKELY(rc == VINF_SUCCESS))
7042	{
7043	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7044	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7045	AssertRCReturn(rc, rc);
7046	Assert(pVmxTransient->cbInstr == 3);
7047
7048	pMixedCtx->rip += pVmxTransient->cbInstr;
7049	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7050
7051	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7052	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7053	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7054	}
7055	else
7056	{
7057	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7058	rc = VERR_EM_INTERPRETER;
7059	}
7060	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7061	return rc;
7062	}
7063
7064
7065	/**
7066	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7067	*/
7068	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7069	{
7070	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7071	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7072	rc \|= hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7073	AssertRCReturn(rc, rc);
7074
7075	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7076	if (RT_LIKELY(rc == VINF_SUCCESS))
7077	{
7078	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7079	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7080	AssertRCReturn(rc, rc);
7081	Assert(pVmxTransient->cbInstr == 2);
7082
7083	pMixedCtx->rip += pVmxTransient->cbInstr;
7084	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7085	}
7086	else
7087	{
7088	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7089	rc = VERR_EM_INTERPRETER;
7090	}
7091	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7092	return rc;
7093	}
7094
7095
7096	/**
7097	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7098	*/
7099	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7100	{
7101	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7102	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7103	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7104	AssertRCReturn(rc, rc);
7105
7106	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7107	rc = VBOXSTRICTRC_VAL(rc2);
7108	if (RT_LIKELY(rc == VINF_SUCCESS))
7109	{
7110	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7111	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7112	AssertRCReturn(rc, rc);
7113
7114	pMixedCtx->rip += pVmxTransient->cbInstr;
7115	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7116	}
7117	else
7118	{
7119	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %RGv failed with %Rrc\n",
7120	pVmxTransient->uExitQualification, rc));
7121	rc = VERR_EM_INTERPRETER;
7122	}
7123	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7124	return rc;
7125	}
7126
7127
7128	/**
7129	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7130	*/
7131	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7132	{
7133	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7134	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7135	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7136	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7137	AssertRCReturn(rc, rc);
7138
7139	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7140	if (RT_LIKELY(rc == VINF_SUCCESS))
7141	{
7142	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7143	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7144	AssertRCReturn(rc, rc);
7145
7146	pMixedCtx->rip += pVmxTransient->cbInstr;
7147	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7148	}
7149	else
7150	{
7151	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7152	rc = VERR_EM_INTERPRETER;
7153	}
7154	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7155	return rc;
7156	}
7157
7158
7159	/**
7160	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7161	*/
7162	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7163	{
7164	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7165	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7166	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7167	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7168	AssertRCReturn(rc, rc);
7169
7170	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7171	rc = VBOXSTRICTRC_VAL(rc2);
7172	if (RT_LIKELY( rc == VINF_SUCCESS
7173	\|\| rc == VINF_EM_HALT))
7174	{
7175	int rc3 = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7176	rc3 \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7177	AssertRCReturn(rc3, rc3);
7178
7179	pMixedCtx->rip += pVmxTransient->cbInstr;
7180	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7181
7182	if ( rc == VINF_EM_HALT
7183	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7184	{
7185	rc = VINF_SUCCESS;
7186	}
7187	}
7188	else
7189	{
7190	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7191	rc = VERR_EM_INTERPRETER;
7192	}
7193	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7194	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7195	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7196	return rc;
7197	}
7198
7199
7200	/**
7201	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7202	*/
7203	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7204	{
7205	/*
7206	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7207	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7208	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7209	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7210	*/
7211	AssertMsgFailed(("Unexpected RSM VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7212	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7213	}
7214
7215
7216	/**
7217	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7218	*/
7219	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7220	{
7221	/*
7222	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7223	* root operation. If we get there there is something funny going on.
7224	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7225	*/
7226	AssertMsgFailed(("Unexpected SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7227	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7228	}
7229
7230
7231	/**
7232	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7233	*/
7234	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7235	{
7236	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7237	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7238	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7239	}
7240
7241
7242	/**
7243	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7244	*/
7245	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7246	{
7247	/*
7248	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7249	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7250	* See Intel spec. 25.3 "Other Causes of VM-exits".
7251	*/
7252	AssertMsgFailed(("Unexpected SIPI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7253	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7254	}
7255
7256
7257	/**
7258	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7259	* VM-exit.
7260	*/
7261	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7262	{
7263	/*
7264	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7265	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7266	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7267	*/
7268	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7269	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7270	}
7271
7272
7273	/**
7274	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7275	* VM-exit.
7276	*/
7277	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7278	{
7279	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7280	return VINF_EM_RESET;
7281	}
7282
7283
7284	/**
7285	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7286	*/
7287	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7288	{
7289	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7290	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7291	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7292	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7293	AssertRCReturn(rc, rc);
7294
7295	pMixedCtx->rip++;
7296	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7297	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7298	rc = VINF_SUCCESS;
7299	else
7300	rc = VINF_EM_HALT;
7301
7302	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7303	return rc;
7304	}
7305
7306
7307	/**
7308	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7309	*/
7310	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7311	{
7312	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7313	return hmR0VmxInjectXcptUD(pVM, pVCpu, pMixedCtx);
7314	}
7315
7316
7317	/**
7318	* VM-exit handler for expiry of the VMX preemption timer.
7319	*/
7320	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7321	{
7322	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7323
7324	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7325	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7326	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7327
7328	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7329	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7330	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7331	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7332	}
7333
7334
7335	/**
7336	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7337	*/
7338	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7339	{
7340	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7341	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7342	/** @todo check if XSETBV is supported by the recompiler. */
7343	return VERR_EM_INTERPRETER;
7344	}
7345
7346
7347	/**
7348	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7349	*/
7350	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7351	{
7352	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7353	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7354	/** @todo implement EMInterpretInvpcid() */
7355	return VERR_EM_INTERPRETER;
7356	}
7357
7358
7359	/**
7360	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7361	* Error VM-exit.
7362	*/
7363	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7364	{
7365	uint32_t uIntrState;
7366	RTHCUINTREG uHCReg;
7367	uint64_t u64Val;
7368
7369	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7370	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7371	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7372	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7373	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7374	AssertRCReturn(rc, rc);
7375
7376	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7377	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7378	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7379	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7380
7381	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7382	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7383	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7384	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7385	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7386	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7387	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7388	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7389	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7390	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7391	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7392	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7393
7394	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7395
7396	return VERR_VMX_INVALID_GUEST_STATE;
7397	}
7398
7399
7400	/**
7401	* VM-exit handler for VM-entry failure due to an MSR-load
7402	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7403	*/
7404	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7405	{
7406	AssertMsgFailed(("Unexpected MSR-load exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7407	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7408	}
7409
7410
7411	/**
7412	* VM-exit handler for VM-entry failure due to a machine-check event
7413	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7414	*/
7415	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7416	{
7417	AssertMsgFailed(("Unexpected machine-check event exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7418	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7419	}
7420
7421
7422	/**
7423	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7424	* theory.
7425	*/
7426	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7427	{
7428	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason,
7429	pVM, pVCpu, pMixedCtx));
7430	return VERR_VMX_UNDEFINED_EXIT_CODE;
7431	}
7432
7433
7434	/**
7435	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7436	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7437	* Conditional VM-exit.
7438	*/
7439	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7440	{
7441	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7442	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7443	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7444	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7445	return VERR_EM_INTERPRETER;
7446	AssertMsgFailed(("Unexpected XDTR access. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7447	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7448	}
7449
7450
7451	/**
7452	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7453	*/
7454	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7455	{
7456	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7457	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7458	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7459	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7460	return VERR_EM_INTERPRETER;
7461	AssertMsgFailed(("Unexpected RDRAND exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7462	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7463	}
7464
7465
7466	/**
7467	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7468	*/
7469	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7470	{
7471	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7472	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7473	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7474	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7475	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7476	AssertRCReturn(rc, rc);
7477
7478	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7479	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7480	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7481	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7482
7483	/* Update RIP and continue guest execution. */
7484	if (RT_LIKELY(rc == VINF_SUCCESS))
7485	{
7486	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7487	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7488	AssertRCReturn(rc, rc);
7489
7490	Assert(pVmxTransient->cbInstr == 2);
7491	pMixedCtx->rip += pVmxTransient->cbInstr;
7492	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7493	}
7494	return rc;
7495	}
7496
7497
7498	/**
7499	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7500	*/
7501	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7502	{
7503	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7504	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7505	AssertRCReturn(rc, rc);
7506	Assert(pVmxTransient->cbInstr == 2);
7507
7508	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7509	if ( pVM->hm.s.fTPRPatchingActive
7510	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7511	{
7512	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7513	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7514	{
7515	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7516	AssertRC(rc);
7517	}
7518
7519	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7520	AssertRCReturn(rc, rc);
7521	pMixedCtx->rip += pVmxTransient->cbInstr;
7522	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7523	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7524	return VINF_SUCCESS;
7525	}
7526
7527	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7528	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS))
7529	{
7530	switch (pMixedCtx->ecx)
7531	{
7532	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7533	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7534	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7535	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FS_BASE_MSR; break;
7536	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_GS_BASE_MSR; break;
7537	}
7538	}
7539	#ifdef DEBUG
7540	else
7541	{
7542	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7543	switch (pMixedCtx->ecx)
7544	{
7545	case MSR_IA32_SYSENTER_CS:
7546	case MSR_IA32_SYSENTER_EIP:
7547	case MSR_IA32_SYSENTER_ESP:
7548	case MSR_K8_FS_BASE:
7549	case MSR_K8_GS_BASE:
7550	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7551	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7552	case MSR_K8_LSTAR:
7553	case MSR_K6_STAR:
7554	case MSR_K8_SF_MASK:
7555	case MSR_K8_TSC_AUX:
7556	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7557	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7558	}
7559	}
7560	#endif
7561
7562	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7563	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7564	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7565	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7566	AssertRCReturn(rc, rc);
7567
7568	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7569	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7570	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7571
7572	/* Update guest-state and continue execution. */
7573	if (RT_LIKELY(rc == VINF_SUCCESS))
7574	{
7575	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7576	AssertRCReturn(rc, rc);
7577
7578	pMixedCtx->rip += pVmxTransient->cbInstr;
7579	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7580
7581	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7582	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7583	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7584	{
7585	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7586	}
7587	}
7588	return rc;
7589	}
7590
7591
7592	/**
7593	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7594	*/
7595	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7596	{
7597	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7598	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7599	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7600	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7601	return VERR_EM_INTERPRETER;
7602	AssertMsgFailed(("Unexpected PAUSE exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7603	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7604	}
7605
7606
7607	/**
7608	* VM-exit handler for when the TPR value is lowered below the specified
7609	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7610	*/
7611	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7612	{
7613	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7614	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7615
7616	/*
7617	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7618	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and
7619	* resume guest execution.
7620	*/
7621	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7622	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7623	return VINF_SUCCESS;
7624	}
7625
7626
7627	/**
7628	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7629	* VM-exit.
7630	*
7631	* @retval VINF_SUCCESS when guest execution can continue.
7632	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7633	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7634	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7635	* recompiler.
7636	*/
7637	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7638	{
7639	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7640	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7641	AssertRCReturn(rc, rc);
7642
7643	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7644	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7645	switch (uAccessType)
7646	{
7647	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7648	{
7649	#if 0
7650	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7651	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7652	#else
7653	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
7654	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7655	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7656	#endif
7657	AssertRCReturn(rc, rc);
7658
7659	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7660	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7661	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7662	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7663
7664	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7665	{
7666	case 0: /* CR0 */
7667	Log(("CR0 write rc=%d\n", rc));
7668	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7669	break;
7670	case 2: /* CR2 */
7671	Log(("CR2 write rc=%d\n", rc));
7672	break;
7673	case 3: /* CR3 */
7674	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7675	Log(("CR3 write rc=%d\n", rc));
7676	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7677	break;
7678	case 4: /* CR4 */
7679	Log(("CR4 write rc=%d\n", rc));
7680	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7681	break;
7682	case 8: /* CR8 */
7683	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7684	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7685	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7686	break;
7687	default:
7688	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7689	break;
7690	}
7691
7692	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7693	break;
7694	}
7695
7696	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7697	{
7698	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7699	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7700	AssertRCReturn(rc, rc);
7701	Assert( !pVM->hm.s.fNestedPaging
7702	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7703	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7704
7705	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7706	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7707	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7708
7709	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7710	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7711	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7712	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7713	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7714	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7715	break;
7716	}
7717
7718	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7719	{
7720	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7721	AssertRCReturn(rc, rc);
7722	rc = EMInterpretCLTS(pVM, pVCpu);
7723	AssertRCReturn(rc, rc);
7724	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7725	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7726	Log(("CRX CLTS write rc=%d\n", rc));
7727	break;
7728	}
7729
7730	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7731	{
7732	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7733	AssertRCReturn(rc, rc);
7734	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7735	if (RT_LIKELY(rc == VINF_SUCCESS))
7736	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7737	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7738	Log(("CRX LMSW write rc=%d\n", rc));
7739	break;
7740	}
7741
7742	default:
7743	{
7744	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7745	return VERR_VMX_UNEXPECTED_EXCEPTION;
7746	}
7747	}
7748
7749	/* Validate possible error codes. */
7750	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3);
7751	if (RT_SUCCESS(rc))
7752	{
7753	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7754	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7755	AssertRCReturn(rc2, rc2);
7756	pMixedCtx->rip += pVmxTransient->cbInstr;
7757	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7758	}
7759
7760	return rc;
7761	}
7762
7763
7764	/**
7765	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7766	* VM-exit.
7767	*/
7768	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7769	{
7770	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7771
7772	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7773	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7774	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7775	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7776	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7777	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7778	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7779	AssertRCReturn(rc, rc);
7780
7781	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7782	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7783	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7784	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7785	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7786	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7787	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7788
7789	/* I/O operation lookup arrays. */
7790	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7791	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7792
7793	const uint32_t cbSize = s_aIOSize[uIOWidth];
7794	const uint32_t cbInstr = pVmxTransient->cbInstr;
7795	if (fIOString)
7796	{
7797	/* INS/OUTS - I/O String instruction. */
7798	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7799	/** @todo for now manually disassemble later optimize by getting the fields from
7800	* the VMCS. */
7801	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7802	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7803	* segment prefix info. */
7804	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7805	if (RT_SUCCESS(rc))
7806	{
7807	if (fIOWrite)
7808	{
7809	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7810	(DISCPUMODE)pDis->uAddrMode, cbSize);
7811	rc = VBOXSTRICTRC_VAL(rc2);
7812	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7813	}
7814	else
7815	{
7816	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7817	(DISCPUMODE)pDis->uAddrMode, cbSize);
7818	rc = VBOXSTRICTRC_VAL(rc2);
7819	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7820	}
7821	}
7822	else
7823	{
7824	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7825	rc = VINF_EM_RAW_EMULATE_INSTR;
7826	}
7827	}
7828	else
7829	{
7830	/* IN/OUT - I/O instruction. */
7831	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7832	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7833	if (fIOWrite)
7834	{
7835	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7836	rc = VBOXSTRICTRC_VAL(rc2);
7837	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7838	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7839	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7840	}
7841	else
7842	{
7843	uint32_t u32Result = 0;
7844	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7845	rc = VBOXSTRICTRC_VAL(rc2);
7846	if (IOM_SUCCESS(rc))
7847	{
7848	/* Save result of I/O IN instr. in AL/AX/EAX. */
7849	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7850	}
7851	else if (rc == VINF_IOM_R3_IOPORT_READ)
7852	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7853	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7854	}
7855	}
7856
7857	if (IOM_SUCCESS(rc))
7858	{
7859	pMixedCtx->rip += cbInstr;
7860	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7861	if (RT_LIKELY(rc == VINF_SUCCESS))
7862	{
7863	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx); /* For DR7. */
7864	AssertRCReturn(rc, rc);
7865
7866	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7867	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7868	{
7869	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7870	for (unsigned i = 0; i < 4; i++)
7871	{
7872	unsigned uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7873	if ( ( uIOPort >= pMixedCtx->dr[i]
7874	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7875	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7876	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7877	{
7878	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7879	uint64_t uDR6 = ASMGetDR6();
7880
7881	/* Clear all breakpoint status flags and set the one we just hit. */
7882	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
7883	uDR6 \|= (uint64_t)RT_BIT(i);
7884
7885	/*
7886	* Note: AMD64 Architecture Programmer's Manual 13.1:
7887	* Bits 15:13 of the DR6 register is never cleared by the processor and must
7888	* be cleared by software after the contents have been read.
7889	*/
7890	ASMSetDR6(uDR6);
7891
7892	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
7893	pMixedCtx->dr[7] &= ~X86_DR7_GD;
7894
7895	/* Paranoia. */
7896	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
7897	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
7898	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
7899
7900	/* Resync DR7 */
7901	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
7902	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7903
7904	/* Inject #DB and get on with guest execution. */
7905	rc = hmR0VmxInjectXcptDB(pVM, pVCpu, pMixedCtx);
7906	AssertRCReturn(rc, rc);
7907	break;
7908	}
7909	}
7910	}
7911	}
7912	}
7913
7914	#ifdef DEBUG
7915	if (rc == VINF_IOM_R3_IOPORT_READ)
7916	Assert(!fIOWrite);
7917	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
7918	Assert(fIOWrite);
7919	else
7920	{
7921	AssertMsg( RT_FAILURE(rc)
7922	\|\| rc == VINF_SUCCESS
7923	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
7924	\|\| rc == VINF_EM_RAW_GUEST_TRAP
7925	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
7926	}
7927	#endif
7928
7929	return rc;
7930	}
7931
7932
7933	/**
7934	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
7935	* VM-exit.
7936	*/
7937	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7938	{
7939	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7940
7941	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
7942	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7943	AssertRCReturn(rc, rc);
7944	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
7945	{
7946	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
7947	AssertRCReturn(rc, rc);
7948	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
7949	{
7950	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
7951	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7952	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7953	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7954	{
7955	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
7956	pVCpu->hm.s.Event.fPending = true;
7957	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
7958	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
7959	AssertRCReturn(rc, rc);
7960	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringErrorCode))
7961	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
7962	else
7963	pVCpu->hm.s.Event.u32ErrCode = 0;
7964	}
7965	}
7966	}
7967	/** @todo Emulate task switch someday, currently just going back to ring-3 for
7968	* emulation. */
7969	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7970	return VERR_EM_INTERPRETER;
7971	}
7972
7973
7974	/**
7975	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
7976	*/
7977	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7978	{
7979	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7980	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
7981	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
7982	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7983	AssertRCReturn(rc, rc);
7984	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
7985	return VINF_EM_DBG_STOP;
7986	}
7987
7988
7989	/**
7990	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
7991	*/
7992	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7993	{
7994	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7995	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7996
7997	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7998	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
7999	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8000	return VINF_SUCCESS;
8001	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8002	return rc;
8003
8004	#if 0
8005	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8006	* just sync the whole thing. */
8007	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8008	#else
8009	/* Aggressive state sync. for now. */
8010	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8011	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8012	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8013	#endif
8014	AssertRCReturn(rc, rc);
8015
8016	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8017	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8018	switch (uAccessType)
8019	{
8020	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8021	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8022	{
8023	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8024	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8025	{
8026	AssertMsgFailed(("hmR0VmxExitApicAccess: can't touch TPR offset while using TPR shadowing.\n"));
8027	}
8028
8029	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8030	GCPhys &= PAGE_BASE_GC_MASK;
8031	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8032	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8033	CPUMCTX2CORE(pMixedCtx), GCPhys);
8034	rc = VBOXSTRICTRC_VAL(rc2);
8035	Log(("ApicAccess %RGp %#x\n", GCPhys, VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8036	if ( rc == VINF_SUCCESS
8037	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8038	\|\| rc == VERR_PAGE_NOT_PRESENT)
8039	{
8040	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8041	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8042	rc = VINF_SUCCESS;
8043	}
8044	break;
8045	}
8046
8047	default:
8048	rc = VINF_EM_RAW_EMULATE_INSTR;
8049	break;
8050	}
8051
8052	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8053	return rc;
8054	}
8055
8056
8057	/**
8058	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8059	* VM-exit.
8060	*/
8061	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8062	{
8063	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8064
8065	/* We should -not- get this VM-exit if the guest is debugging. */
8066	if (CPUMIsGuestDebugStateActive(pVCpu))
8067	{
8068	AssertMsgFailed(("Unexpected MOV DRx exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
8069	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8070	}
8071
8072	int rc = VERR_INTERNAL_ERROR_5;
8073	if ( !DBGFIsStepping(pVCpu)
8074	&& !CPUMIsHyperDebugStateActive(pVCpu))
8075	{
8076	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
8077
8078	/* Don't intercept MOV DRx. */
8079	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8080	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8081	AssertRCReturn(rc, rc);
8082
8083	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8084	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8085	AssertRC(rc);
8086	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8087
8088	#ifdef VBOX_WITH_STATISTICS
8089	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8090	AssertRCReturn(rc, rc);
8091	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8092	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8093	else
8094	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8095	#endif
8096	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8097	return VINF_SUCCESS;
8098	}
8099
8100	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
8101	* time and restore DRx registers afterwards */
8102	/*
8103	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8104	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8105	*/
8106	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8107	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8108	AssertRCReturn(rc, rc);
8109
8110	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8111	{
8112	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8113	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8114	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8115	if (RT_SUCCESS(rc))
8116	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8117	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8118	}
8119	else
8120	{
8121	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8122	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8123	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8124	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8125	}
8126
8127	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8128	if (RT_SUCCESS(rc))
8129	{
8130	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8131	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8132	AssertRCReturn(rc2, rc2);
8133	pMixedCtx->rip += pVmxTransient->cbInstr;
8134	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8135	}
8136	return rc;
8137	}
8138
8139
8140	/**
8141	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8142	* Conditional VM-exit.
8143	*/
8144	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8145	{
8146	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8147	Assert(pVM->hm.s.fNestedPaging);
8148
8149	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8150	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8151	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8152	return VINF_SUCCESS;
8153	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8154	return rc;
8155
8156	RTGCPHYS GCPhys = 0;
8157	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8158
8159	#if 0
8160	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
8161	#else
8162	/* Aggressive state sync. for now. */
8163	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8164	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8165	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8166	#endif
8167	AssertRCReturn(rc, rc);
8168
8169	/*
8170	* If we succeed, resume guest execution.
8171	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8172	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8173	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8174	* weird case. See @bugref{6043}.
8175	*/
8176	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8177	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8178	rc = VBOXSTRICTRC_VAL(rc2);
8179	if ( rc == VINF_SUCCESS
8180	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8181	\|\| rc == VERR_PAGE_NOT_PRESENT)
8182	{
8183	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8184	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8185	return VINF_SUCCESS;
8186	}
8187	return rc;
8188	}
8189
8190
8191	/**
8192	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8193	* VM-exit.
8194	*/
8195	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8196	{
8197	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8198	Assert(pVM->hm.s.fNestedPaging);
8199
8200	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8201	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8202	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8203	return VINF_SUCCESS;
8204	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8205	return rc;
8206
8207	RTGCPHYS GCPhys = 0;
8208	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8209	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8210	#if 0
8211	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
8212	#else
8213	/* Aggressive state sync. for now. */
8214	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8215	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8216	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8217	#endif
8218	AssertRCReturn(rc, rc);
8219
8220	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8221	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8222
8223	RTGCUINT uErrorCode = 0;
8224	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8225	uErrorCode \|= X86_TRAP_PF_ID;
8226	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8227	uErrorCode \|= X86_TRAP_PF_RW;
8228	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8229	uErrorCode \|= X86_TRAP_PF_P;
8230
8231	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8232	TRPMSetErrorCode(pVCpu, uErrorCode);
8233	TRPMSetFaultAddress(pVCpu, GCPhys);
8234
8235	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8236	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8237
8238	/* Handle the pagefault trap for the nested shadow table. */
8239	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8240	TRPMResetTrap(pVCpu);
8241
8242	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8243	if ( rc == VINF_SUCCESS
8244	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8245	\|\| rc == VERR_PAGE_NOT_PRESENT)
8246	{
8247	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8248	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8249	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8250	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8251	return VINF_SUCCESS;
8252	}
8253
8254	Log(("EPT return to ring-3 rc=%d\n"));
8255	return rc;
8256	}
8257
8258
8259	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8260	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8261	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8262	/**
8263	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8264	*/
8265	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8266	{
8267	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8268
8269	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
8270	AssertRCReturn(rc, rc);
8271	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8272
8273	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8274	{
8275	/* Old-style FPU error reporting needs some extra work. */
8276	/** @todo don't fall back to the recompiler, but do it manually. */
8277	return VERR_EM_INTERPRETER;
8278	}
8279	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8280	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8281	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8282	AssertRCReturn(rc, rc);
8283	return rc;
8284	}
8285
8286
8287	/**
8288	* VM-exit exception handler for #BP (Breakpoint exception).
8289	*/
8290	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8291	{
8292	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8293
8294	/** @todo Try optimize this by not saving the entire guest state unless
8295	* really needed. */
8296	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8297	AssertRCReturn(rc, rc);
8298	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8299
8300	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8301	if (rc == VINF_EM_RAW_GUEST_TRAP)
8302	{
8303	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8304	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8305	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8306	AssertRCReturn(rc, rc);
8307
8308	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8309	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8310	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8311	AssertRCReturn(rc, rc);
8312	}
8313
8314	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8315	return rc;
8316	}
8317
8318
8319	/**
8320	* VM-exit exception handler for #DB (Debug exception).
8321	*/
8322	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8323	{
8324	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8325
8326	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8327	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8328	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8329	AssertRCReturn(rc, rc);
8330
8331	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8332	uint64_t uDR6 = X86_DR6_INIT_VAL;
8333	uDR6 \|= (pVmxTransient->uExitQualification
8334	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8335	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8337	if (rc == VINF_EM_RAW_GUEST_TRAP)
8338	{
8339	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8340	pMixedCtx->dr[6] = uDR6;
8341
8342	if (CPUMIsGuestDebugStateActive(pVCpu))
8343	ASMSetDR6(pMixedCtx->dr[6]);
8344
8345	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8346	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8347
8348	/* Paranoia. */
8349	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8350	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8351	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8352
8353	/* Resync DR7. */
8354	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8355
8356	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8357	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8358	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8359	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8360	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8361	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8362	AssertRCReturn(rc,rc);
8363	return rc;
8364	}
8365	/* Return to ring 3 to deal with the debug exit code. */
8366	return rc;
8367	}
8368
8369
8370	/**
8371	* VM-exit exception handler for #NM (Device-not-available exception: floating
8372	* point exception).
8373	*/
8374	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8375	{
8376	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8377
8378	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8379	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8380	#endif
8381
8382	/* We require CR0 and EFER. EFER is always up-to-date. */
8383	int rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8384	AssertRCReturn(rc, rc);
8385
8386	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8387	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8388	if (rc == VINF_SUCCESS)
8389	{
8390	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8391	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8392	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8393	return VINF_SUCCESS;
8394	}
8395
8396	/* Forward #NM to the guest. */
8397	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8398	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8399	AssertRCReturn(rc, rc);
8400	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8401	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8402	pVmxTransient->cbInstr, 0 /* error code */);
8403	AssertRCReturn(rc, rc);
8404	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8405	return rc;
8406	}
8407
8408
8409	/**
8410	* VM-exit exception handler for #GP (General-protection exception).
8411	*
8412	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8413	*/
8414	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8415	{
8416	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8417
8418	int rc = VERR_INTERNAL_ERROR_5;
8419
8420	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8421	{
8422	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8423	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8424	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8425	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8426	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8427	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8428	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8429	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8430	AssertRCReturn(rc, rc);
8431	return rc;
8432	#else
8433	/* We don't intercept #GP. */
8434	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8435	return VERR_VMX_UNEXPECTED_EXCEPTION;
8436	#endif
8437	}
8438
8439	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8440	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
8441
8442	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8443	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8444	AssertRCReturn(rc, rc);
8445
8446	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8447	unsigned int cbOp = 0;
8448	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8449	if (RT_SUCCESS(rc))
8450	{
8451	rc = VINF_SUCCESS;
8452	Assert(cbOp == pDis->cbInstr);
8453	Log2(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8454	switch (pDis->pCurInstr->uOpcode)
8455	{
8456	case OP_CLI:
8457	pMixedCtx->eflags.Bits.u1IF = 0;
8458	pMixedCtx->rip += pDis->cbInstr;
8459	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8460	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8461	break;
8462
8463	case OP_STI:
8464	pMixedCtx->eflags.Bits.u1IF = 1;
8465	pMixedCtx->rip += pDis->cbInstr;
8466	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8467	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8468	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8469	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8470	break;
8471
8472	case OP_HLT:
8473	rc = VINF_EM_HALT;
8474	pMixedCtx->rip += pDis->cbInstr;
8475	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8476	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8477	break;
8478
8479	case OP_POPF:
8480	{
8481	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8482	uint32_t cbParm = 0;
8483	uint32_t uMask = 0;
8484	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8485	{
8486	cbParm = 4;
8487	uMask = 0xffffffff;
8488	}
8489	else
8490	{
8491	cbParm = 2;
8492	uMask = 0xffff;
8493	}
8494
8495	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8496	RTGCPTR GCPtrStack = 0;
8497	X86EFLAGS uEflags;
8498	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8499	&GCPtrStack);
8500	if (RT_SUCCESS(rc))
8501	{
8502	Assert(sizeof(uEflags.u32) >= cbParm);
8503	uEflags.u32 = 0;
8504	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8505	}
8506	if (RT_FAILURE(rc))
8507	{
8508	rc = VERR_EM_INTERPRETER;
8509	break;
8510	}
8511	Log(("POPF %x -> %RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8512	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8513	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8514	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8515	pMixedCtx->eflags.Bits.u1RF = 0;
8516	pMixedCtx->esp += cbParm;
8517	pMixedCtx->esp &= uMask;
8518	pMixedCtx->rip += pDis->cbInstr;
8519	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8520	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8521	break;
8522	}
8523
8524	case OP_PUSHF:
8525	{
8526	uint32_t cbParm = 0;
8527	uint32_t uMask = 0;
8528	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8529	{
8530	cbParm = 4;
8531	uMask = 0xffffffff;
8532	}
8533	else
8534	{
8535	cbParm = 2;
8536	uMask = 0xffff;
8537	}
8538
8539	/* Get the stack pointer & push the contents of eflags onto the stack. */
8540	RTGCPTR GCPtrStack = 0;
8541	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8542	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8543	if (RT_FAILURE(rc))
8544	{
8545	rc = VERR_EM_INTERPRETER;
8546	break;
8547	}
8548	X86EFLAGS uEflags;
8549	uEflags = pMixedCtx->eflags;
8550	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8551	uEflags.Bits.u1RF = 0;
8552	uEflags.Bits.u1VM = 0;
8553
8554	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8555	if (RT_FAILURE(rc))
8556	{
8557	rc = VERR_EM_INTERPRETER;
8558	break;
8559	}
8560	Log(("PUSHF %x -> %RGv\n", uEflags.u, GCPtrStack));
8561	pMixedCtx->esp -= cbParm;
8562	pMixedCtx->esp &= uMask;
8563	pMixedCtx->rip += pDis->cbInstr;
8564	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8565	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8566	break;
8567	}
8568
8569	case OP_IRET:
8570	{
8571	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8572	* instruction reference. */
8573	RTGCPTR GCPtrStack = 0;
8574	uint32_t uMask = 0xffff;
8575	uint16_t aIretFrame[3];
8576	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8577	{
8578	rc = VERR_EM_INTERPRETER;
8579	break;
8580	}
8581	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8582	&GCPtrStack);
8583	if (RT_SUCCESS(rc))
8584	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8585	if (RT_FAILURE(rc))
8586	{
8587	rc = VERR_EM_INTERPRETER;
8588	break;
8589	}
8590	pMixedCtx->eip = 0;
8591	pMixedCtx->ip = aIretFrame[0];
8592	pMixedCtx->cs.Sel = aIretFrame[1];
8593	pMixedCtx->cs.ValidSel = aIretFrame[1];
8594	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8595	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8596	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8597	pMixedCtx->sp += sizeof(aIretFrame);
8598	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8599	\| HM_CHANGED_GUEST_RFLAGS;
8600	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8601	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8602	break;
8603	}
8604
8605	case OP_INT:
8606	{
8607	uint16_t uVector = pDis->Param1.uValue & 0xff;
8608	rc = hmR0VmxInjectIntN(pVM, pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8609	AssertRCReturn(rc, rc);
8610	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8611	break;
8612	}
8613
8614	case OP_INTO:
8615	{
8616	if (pMixedCtx->eflags.Bits.u1OF)
8617	{
8618	rc = hmR0VmxInjectXcptOF(pVM, pVCpu, pMixedCtx, pDis->cbInstr);
8619	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8620	}
8621	break;
8622	}
8623
8624	default:
8625	{
8626	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8627	EMCODETYPE_SUPERVISOR);
8628	rc = VBOXSTRICTRC_VAL(rc2);
8629	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8630	Log2(("#GP rc=%Rrc\n", rc));
8631	break;
8632	}
8633	}
8634	}
8635	else
8636	rc = VERR_EM_INTERPRETER;
8637
8638	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT
8639	\|\| rc == VINF_EM_RESET /* injection caused triple fault */,
8640	("#GP Unexpected rc=%Rrc\n", rc));
8641	return rc;
8642	}
8643
8644
8645	/**
8646	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8647	* the exception reported in the VMX transient structure back into the VM.
8648	*
8649	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8650	* VMX transient structure to be up-to-date.
8651	*/
8652	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8653	{
8654	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8655
8656	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8657	hmR0VmxCheckExitDueToEventDelivery(). */
8658	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8659	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8660	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8661	AssertRCReturn(rc, rc);
8662	return rc;
8663	}
8664
8665
8666	/**
8667	* VM-exit exception handler for #PF (Page-fault exception).
8668	*/
8669	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8670	{
8671	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8672
8673	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8674	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8675	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8676	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8677	AssertRCReturn(rc, rc);
8678
8679	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8680	if (pVM->hm.s.fNestedPaging)
8681	{
8682	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8683	{
8684	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8685	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8686	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8687	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8688	AssertRCReturn(rc, rc);
8689	}
8690	else
8691	{
8692	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8693	Assert(!pVCpu->hm.s.Event.fPending);
8694	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8695	}
8696	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8697	return rc;
8698	}
8699	#else
8700	Assert(!pVM->hm.s.fNestedPaging);
8701	#endif
8702
8703	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8704	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8705	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8706	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8707	AssertRCReturn(rc, rc);
8708	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8709	if ( pVM->hm.s.fTRPPatchingAllowed
8710	&& pVM->hm.s.pGuestPatchMem
8711	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8712	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8713	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8714	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8715	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8716	{
8717	RTGCPHYS GCPhys;
8718	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8719	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8720	if ( rc == VINF_SUCCESS
8721	&& GCPhys == GCPhysApicBase)
8722	{
8723	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8724	AssertRCReturn(rc, rc);
8725
8726	/* Only attempt to patch the instruction once. */
8727	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8728	if (!pPatch)
8729	return VINF_EM_HM_PATCH_TPR_INSTR;
8730	}
8731	}
8732	#endif
8733
8734	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8735	TRPMSetFaultAddress(pVCpu, pVmxTransient->uExitQualification);
8736	TRPMSetErrorCode(pVCpu, pVmxTransient->uExitIntrErrorCode);
8737
8738	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8739	AssertRCReturn(rc, rc);
8740
8741	/* Forward it to the trap handler first. */
8742	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8743	(RTGCPTR)pVmxTransient->uExitQualification);
8744
8745	Log(("#PF: cr2=%RGv cs:rip=%04x:%RGv errorcode %#RX32 rc=%d\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8746	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, rc));
8747
8748	if (rc == VINF_SUCCESS)
8749	{
8750	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8751	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8752	* memory? We don't update the whole state here... */
8753	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8754	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8755
8756	TRPMResetTrap(pVCpu);
8757	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8758	return rc;
8759	}
8760	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8761	{
8762	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8763	{
8764	/* It's a guest page fault and needs to be reflected to the guest. */
8765	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8766	TRPMResetTrap(pVCpu);
8767	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8768	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8769	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8770	pVmxTransient->cbInstr, uGstErrorCode);
8771	AssertRCReturn(rc, rc);
8772	}
8773	else
8774	{
8775	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8776	Assert(!pVCpu->hm.s.Event.fPending);
8777	TRPMResetTrap(pVCpu);
8778	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8779	}
8780	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8781	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8782	return rc;
8783	}
8784
8785	TRPMResetTrap(pVCpu);
8786	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8787	return rc;
8788	}
8789

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45478

Download in other formats: