HMVMXR0.cpp@ 45494

Last change on this file since 45494 was 45487, checked in by vboxsync, 12 years ago
VMMR0/HMVMXR0: Fix LDTR usable bit, fixes NetBSD guests.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 370.0 KB

Line
1	/* $Id: HMVMXR0.cpp 45487 2013-04-11 16:01:22Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
39	#endif
40
41	/*******************************************************************************
42	* Defined Constants And Macros *
43	*******************************************************************************/
44	#if defined(RT_ARCH_AMD64)
45	# define VMX_IS_64BIT_HOST_MODE() (true)
46	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
47	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
48	#else
49	# define VMX_IS_64BIT_HOST_MODE() (false)
50	#endif
51
52	#define VMX_SEL_UNUSABLE RT_BIT(16)
53
54	/**
55	* Updated-guest-state flags.
56	*/
57	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
58	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
59	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
60	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
61	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
62	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
63	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
64	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
65	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
66	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
67	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
68	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
69	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
70	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
71	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
72	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
73	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
74	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
75	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(18)
76	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(19)
77	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(20)
78	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
79	\| VMX_UPDATED_GUEST_RIP \
80	\| VMX_UPDATED_GUEST_RSP \
81	\| VMX_UPDATED_GUEST_RFLAGS \
82	\| VMX_UPDATED_GUEST_CR0 \
83	\| VMX_UPDATED_GUEST_CR3 \
84	\| VMX_UPDATED_GUEST_CR4 \
85	\| VMX_UPDATED_GUEST_GDTR \
86	\| VMX_UPDATED_GUEST_IDTR \
87	\| VMX_UPDATED_GUEST_LDTR \
88	\| VMX_UPDATED_GUEST_TR \
89	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
90	\| VMX_UPDATED_GUEST_DEBUG \
91	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
92	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
93	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
94	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
95	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
96	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
97	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
98	\| VMX_UPDATED_GUEST_APIC_STATE)
99
100	/**
101	* Flags to skip redundant reads of some common VMCS fields.
102	*/
103	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
104	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
105	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
106	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
107	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
108	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
109
110	/*
111	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
112	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
113	* we have Nested Paging support.
114	*/
115	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
116	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
117	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
118	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
119	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
120	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
121	\| RT_BIT(X86_XCPT_XF))
122
123	/* Maximum VM-instruction error number. */
124	#define VMX_INSTR_ERROR_MAX 28
125
126	/*******************************************************************************
127	* Structures and Typedefs *
128	*******************************************************************************/
129	/**
130	* A state structure for holding miscellaneous information across
131	* VMX non-root operation and restored after the transition.
132	*/
133	typedef struct VMXTRANSIENT
134	{
135	/** The host's rflags/eflags. */
136	RTCCUINTREG uEFlags;
137	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
138	uint64_t u64LStarMsr;
139	/** The guest's TPR value used for TPR shadowing. */
140	uint8_t u8GuestTpr;
141
142	/** The basic VM-exit reason. */
143	uint16_t uExitReason;
144	/** The VM-exit exit qualification. */
145	RTGCUINTPTR uExitQualification;
146	/** The VM-exit interruption error code. */
147	uint32_t uExitIntrErrorCode;
148
149	/** The VM-exit interruption-information field. */
150	uint32_t uExitIntrInfo;
151	/** Whether the VM-entry failed or not. */
152	bool fVMEntryFailed;
153	/** The VM-exit instruction-length field. */
154	uint32_t cbInstr;
155
156	/** The VM-entry interruption-information field. */
157	uint32_t uEntryIntrInfo;
158	/** The VM-entry exception error code field. */
159	uint32_t uEntryXcptErrorCode;
160	/** The VM-entry instruction length field. */
161	uint32_t cbEntryInstr;
162
163	/** IDT-vectoring information field. */
164	uint32_t uIdtVectoringInfo;
165	/** IDT-vectoring error code. */
166	uint32_t uIdtVectoringErrorCode;
167
168	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
169	uint32_t fVmcsFieldsRead;
170	/** Whether TSC-offsetting should be setup before VM-entry. */
171	bool fUpdateTscOffsettingAndPreemptTimer;
172	/** Whether the VM-exit was caused by a page-fault during delivery of a
173	* contributary exception or a page-fault. */
174	bool fVectoringPF;
175	} VMXTRANSIENT, *PVMXTRANSIENT;
176
177	/**
178	* MSR-bitmap read permissions.
179	*/
180	typedef enum VMXMSREXITREAD
181	{
182	/** Reading this MSR causes a VM-exit. */
183	VMXMSREXIT_INTERCEPT_READ = 0xb,
184	/** Reading this MSR does not cause a VM-exit. */
185	VMXMSREXIT_PASSTHRU_READ
186	} VMXMSREXITREAD;
187
188	/**
189	* MSR-bitmap write permissions.
190	*/
191	typedef enum VMXMSREXITWRITE
192	{
193	/** Writing to this MSR causes a VM-exit. */
194	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
195	/** Writing to this MSR does not cause a VM-exit. */
196	VMXMSREXIT_PASSTHRU_WRITE
197	} VMXMSREXITWRITE;
198
199	/*******************************************************************************
200	* Internal Functions *
201	*******************************************************************************/
202	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
203	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
204	uint32_t cbInstr, uint32_t u32ErrCode);
205	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
206	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
207	#endif
208	#if 0
209	DECLINLINE(int) hmR0VmxHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
210	unsigned rcReason);
211	#endif
212
213	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
214	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
215	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
216	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
217	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
218	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
219	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
220	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
221	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
222	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
223	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
224	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264
265	/*******************************************************************************
266	* Global Variables *
267	*******************************************************************************/
268	/** @todo Move this to hm_vmx.h. */
269	/**
270	* VM-exit handler.
271	*
272	* @returns VBox status code.
273	* @param pVM Pointer to the VM.
274	* @param pVCpu Pointer to the VMCPU.
275	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
276	* out-of-sync. Make sure to update the required
277	* fields before using them.
278	* @param pVmxTransient Pointer to the VMX-transient structure.
279	*/
280	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	/** Pointer to VM-exit handler. */
282	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
283
284	static const PFNVMEXITHANDLER s_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
285	{
286	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
287	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
288	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
289	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
290	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
291	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
292	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
293	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
294	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
295	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
296	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
297	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
298	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
299	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
300	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
301	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
302	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
303	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
304	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitInjectXcptUD,
305	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitInjectXcptUD,
306	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitInjectXcptUD,
307	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitInjectXcptUD,
308	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitInjectXcptUD,
309	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitInjectXcptUD,
310	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitInjectXcptUD,
311	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitInjectXcptUD,
312	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitInjectXcptUD,
313	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitInjectXcptUD,
314	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
315	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
316	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
317	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
318	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
319	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
320	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
321	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
322	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
323	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
324	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
325	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
326	/* 40 UNDEFINED */ hmR0VmxExitPause,
327	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
328	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
329	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
330	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
331	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
332	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
333	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
334	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
335	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
336	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitInjectXcptUD,
337	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
338	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
339	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInjectXcptUD,
340	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
341	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
342	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
343	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
344	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
345	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitInjectXcptUD
346	};
347
348	static const char* const s_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
349	{
350	/* 0 */ "(Not Used)",
351	/* 1 */ "VMCALL executed in VMX root operation.",
352	/* 2 */ "VMCLEAR with invalid physical address.",
353	/* 3 */ "VMCLEAR with VMXON pointer.",
354	/* 4 */ "VMLAUNCH with non-clear VMCS.",
355	/* 5 */ "VMRESUME with non-launched VMCS.",
356	/* 6 */ "VMRESUME after VMXOFF",
357	/* 7 */ "VM entry with invalid control fields.",
358	/* 8 */ "VM entry with invalid host state fields.",
359	/* 9 */ "VMPTRLD with invalid physical address.",
360	/* 10 */ "VMPTRLD with VMXON pointer.",
361	/* 11 */ "VMPTRLD with incorrect revision identifier.",
362	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
363	/* 13 */ "VMWRITE to read-only VMCS component.",
364	/* 14 */ "(Not Used)",
365	/* 15 */ "VMXON executed in VMX root operation.",
366	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
367	/* 17 */ "VM entry with non-launched executing VMCS.",
368	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
369	/* 19 */ "VMCALL with non-clear VMCS.",
370	/* 20 */ "VMCALL with invalid VM-exit control fields.",
371	/* 21 */ "(Not Used)",
372	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
373	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
374	/* 24 */ "VMCALL with invalid SMM-monitor features.",
375	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
376	/* 26 */ "VM entry with events blocked by MOV SS.",
377	/* 27 */ "(Not Used)",
378	/* 28 */ "Invalid operand to INVEPT/INVVPID."
379	};
380
381
382	/**
383	* Updates the VM's last error record. If there was a VMX instruction error,
384	* reads the error data from the VMCS and updates VCPU's last error record as
385	* well.
386	*
387	* @param pVM Pointer to the VM.
388	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
389	* VERR_VMX_UNABLE_TO_START_VM or
390	* VERR_VMX_INVALID_VMCS_FIELD).
391	* @param rc The error code.
392	*/
393	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
394	{
395	AssertPtr(pVM);
396	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
397	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
398	{
399	AssertPtrReturnVoid(pVCpu);
400	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
401	}
402	pVM->hm.s.lLastError = rc;
403	}
404
405
406	/**
407	* Reads the VM-entry interruption-information field from the VMCS into the VMX
408	* transient structure.
409	*
410	* @returns VBox status code.
411	* @param pVmxTransient Pointer to the VMX transient structure.
412	*
413	* @remarks No-long-jump zone!!!
414	*/
415	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
416	{
417	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
418	AssertRCReturn(rc, rc);
419	return VINF_SUCCESS;
420	}
421
422
423	/**
424	* Reads the VM-entry exception error code field from the VMCS into
425	* the VMX transient structure.
426	*
427	* @returns VBox status code.
428	* @param pVmxTransient Pointer to the VMX transient structure.
429	*
430	* @remarks No-long-jump zone!!!
431	*/
432	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
433	{
434	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
435	AssertRCReturn(rc, rc);
436	return VINF_SUCCESS;
437	}
438
439
440	/**
441	* Reads the VM-entry exception error code field from the VMCS into
442	* the VMX transient structure.
443	*
444	* @returns VBox status code.
445	* @param pVCpu Pointer to the VMCPU.
446	* @param pVmxTransient Pointer to the VMX transient structure.
447	*
448	* @remarks No-long-jump zone!!!
449	*/
450	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
451	{
452	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
453	AssertRCReturn(rc, rc);
454	return VINF_SUCCESS;
455	}
456
457
458	/**
459	* Reads the VM-exit interruption-information field from the VMCS into the VMX
460	* transient structure.
461	*
462	* @returns VBox status code.
463	* @param pVCpu Pointer to the VMCPU.
464	* @param pVmxTransient Pointer to the VMX transient structure.
465	*/
466	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
467	{
468	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
469	{
470	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
471	AssertRCReturn(rc, rc);
472	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
473	}
474	return VINF_SUCCESS;
475	}
476
477
478	/**
479	* Reads the VM-exit interruption error code from the VMCS into the VMX
480	* transient structure.
481	*
482	* @returns VBox status code.
483	* @param pVCpu Pointer to the VMCPU.
484	* @param pVmxTransient Pointer to the VMX transient structure.
485	*/
486	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
487	{
488	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
489	{
490	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
491	AssertRCReturn(rc, rc);
492	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
493	}
494	return VINF_SUCCESS;
495	}
496
497
498	/**
499	* Reads the VM-exit instruction length field from the VMCS into the VMX
500	* transient structure.
501	*
502	* @returns VBox status code.
503	* @param pVCpu Pointer to the VMCPU.
504	* @param pVmxTransient Pointer to the VMX transient structure.
505	*/
506	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
507	{
508	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
509	{
510	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
511	AssertRCReturn(rc, rc);
512	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
513	}
514	return VINF_SUCCESS;
515	}
516
517
518	/**
519	* Reads the exit qualification from the VMCS into the VMX transient structure.
520	*
521	* @returns VBox status code.
522	* @param pVCpu Pointer to the VMCPU.
523	* @param pVmxTransient Pointer to the VMX transient structure.
524	*/
525	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
526	{
527	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
528	{
529	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
530	AssertRCReturn(rc, rc);
531	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
532	}
533	return VINF_SUCCESS;
534	}
535
536
537	/**
538	* Reads the IDT-vectoring information field from the VMCS into the VMX
539	* transient structure.
540	*
541	* @returns VBox status code.
542	* @param pVmxTransient Pointer to the VMX transient structure.
543	*
544	* @remarks No-long-jump zone!!!
545	*/
546	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
547	{
548	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
549	{
550	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
551	AssertRCReturn(rc, rc);
552	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
553	}
554	return VINF_SUCCESS;
555	}
556
557
558	/**
559	* Reads the IDT-vectoring error code from the VMCS into the VMX
560	* transient structure.
561	*
562	* @returns VBox status code.
563	* @param pVmxTransient Pointer to the VMX transient structure.
564	*/
565	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
566	{
567	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
568	{
569	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
570	AssertRCReturn(rc, rc);
571	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
572	}
573	return VINF_SUCCESS;
574	}
575
576
577	/**
578	* Enters VMX root mode operation on the current CPU.
579	*
580	* @returns VBox status code.
581	* @param pVM Pointer to the VM (optional, can be NULL, after
582	* a resume).
583	* @param HCPhysCpuPage Physical address of the VMXON region.
584	* @param pvCpuPage Pointer to the VMXON region.
585	*/
586	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
587	{
588	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
589	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
590
591	if (pVM)
592	{
593	/* Write the VMCS revision dword to the VMXON region. */
594	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
595	}
596
597	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
598	RTCCUINTREG fFlags = ASMIntDisableFlags();
599
600	/* Enable the VMX bit in CR4 if necessary. */
601	RTCCUINTREG uCr4 = ASMGetCR4();
602	if (!(uCr4 & X86_CR4_VMXE))
603	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
604
605	/* Enter VMX root mode. */
606	int rc = VMXEnable(HCPhysCpuPage); /** @todo This would #GP(0) if we are already in VMX root mode... try skip it? */
607	if (RT_FAILURE(rc))
608	ASMSetCR4(uCr4);
609
610	/* Restore interrupts. */
611	ASMSetFlags(fFlags);
612	return rc;
613	}
614
615
616	/**
617	* Exits VMX root mode operation on the current CPU.
618	*
619	* @returns VBox status code.
620	*/
621	static int hmR0VmxLeaveRootMode(void)
622	{
623	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
624	RTCCUINTREG fFlags = ASMIntDisableFlags();
625	int rc = VINF_SUCCESS;
626
627	/* If we're for some reason not in VMX root mode, then don't leave it. */
628	if (ASMGetCR4() & X86_CR4_VMXE)
629	{
630	/* Exit VMX root mode and clear the VMX bit in CR4 */
631	VMXDisable();
632	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
633	}
634	else
635	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
636
637	/* Restore interrupts. */
638	ASMSetFlags(fFlags);
639	return rc;
640	}
641
642
643	/**
644	* Allocates and maps one physically contiguous page. The allocated page is
645	* zero'd out. (Used by various VT-x structures).
646	*
647	* @returns IPRT status code.
648	* @param pMemObj Pointer to the ring-0 memory object.
649	* @param ppVirt Where to store the virtual address of the
650	* allocation.
651	* @param pPhys Where to store the physical address of the
652	* allocation.
653	*/
654	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
655	{
656	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
657	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
658	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
659
660	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
661	if (RT_FAILURE(rc))
662	return rc;
663	ppVirt = RTR0MemObjAddress(pMemObj);
664	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
665	ASMMemZero32(*ppVirt, PAGE_SIZE);
666	return VINF_SUCCESS;
667	}
668
669
670	/**
671	* Frees and unmaps an allocated physical page.
672	*
673	* @param pMemObj Pointer to the ring-0 memory object.
674	* @param ppVirt Where to re-initialize the virtual address of
675	* allocation as 0.
676	* @param pHCPhys Where to re-initialize the physical address of the
677	* allocation as 0.
678	*/
679	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
680	{
681	AssertPtr(pMemObj);
682	AssertPtr(ppVirt);
683	AssertPtr(pHCPhys);
684	if (*pMemObj != NIL_RTR0MEMOBJ)
685	{
686	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
687	AssertRC(rc);
688	*pMemObj = NIL_RTR0MEMOBJ;
689	*ppVirt = 0;
690	*pHCPhys = 0;
691	}
692	}
693
694
695	/**
696	* Worker function to free VT-x related structures.
697	*
698	* @returns IPRT status code.
699	* @param pVM Pointer to the VM.
700	*/
701	static void hmR0VmxStructsFree(PVM pVM)
702	{
703	for (VMCPUID i = 0; i < pVM->cCpus; i++)
704	{
705	PVMCPU pVCpu = &pVM->aCpus[i];
706	AssertPtr(pVCpu);
707
708	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
709	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
710
711	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
712	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
713
714	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
715	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
716	}
717
718	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
719	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
720	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
721	#endif
722	}
723
724
725	/**
726	* Worker function to allocate VT-x related VM structures.
727	*
728	* @returns IPRT status code.
729	* @param pVM Pointer to the VM.
730	*/
731	static int hmR0VmxStructsAlloc(PVM pVM)
732	{
733	/*
734	* Initialize members up-front so we can cleanup properly on allocation failure.
735	*/
736	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
737	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
738	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
739	pVM->hm.s.vmx.HCPhys##a_Name = 0;
740
741	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
742	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
743	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
744	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
745
746	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
747	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
748	#endif
749	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
750
751	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
752	for (VMCPUID i = 0; i < pVM->cCpus; i++)
753	{
754	PVMCPU pVCpu = &pVM->aCpus[i];
755	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
756	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
757	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
758	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
759	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
760	}
761	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
762	#undef VMXLOCAL_INIT_VM_MEMOBJ
763
764	/*
765	* Allocate all the VT-x structures.
766	*/
767	int rc = VINF_SUCCESS;
768	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
769	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
770	if (RT_FAILURE(rc))
771	goto cleanup;
772	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
773	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
774	#endif
775
776	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
777	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
778	{
779	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
780	&pVM->hm.s.vmx.HCPhysApicAccess);
781	if (RT_FAILURE(rc))
782	goto cleanup;
783	}
784
785	/*
786	* Initialize per-VCPU VT-x structures.
787	*/
788	for (VMCPUID i =0; i < pVM->cCpus; i++)
789	{
790	PVMCPU pVCpu = &pVM->aCpus[i];
791	AssertPtr(pVCpu);
792
793	/* Allocate the VM control structure (VMCS). */
794	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
795	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
796	if (RT_FAILURE(rc))
797	goto cleanup;
798
799	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
800	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
801	{
802	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
803	&pVCpu->hm.s.vmx.HCPhysVirtApic);
804	if (RT_FAILURE(rc))
805	goto cleanup;
806	}
807
808	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
809	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
810	{
811	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
812	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
813	if (RT_FAILURE(rc))
814	goto cleanup;
815	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
816	}
817
818	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
819	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
820	if (RT_FAILURE(rc))
821	goto cleanup;
822
823	/* Allocate the VM-exit MSR-load page for the host MSRs. */
824	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
825	if (RT_FAILURE(rc))
826	goto cleanup;
827	}
828
829	return VINF_SUCCESS;
830
831	cleanup:
832	hmR0VmxStructsFree(pVM);
833	return rc;
834	}
835
836
837	/**
838	* Does global VT-x initialization (called during module initialization).
839	*
840	* @returns VBox status code.
841	*/
842	VMMR0DECL(int) VMXR0GlobalInit(void)
843	{
844	/* Setup the main VM exit handlers. */
845	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(s_apfnVMExitHandlers));
846	#ifdef DEBUG
847	for (unsigned i = 0; i < RT_ELEMENTS(s_apfnVMExitHandlers); i++)
848	Assert(s_apfnVMExitHandlers[i]);
849	#endif
850	return VINF_SUCCESS;
851	}
852
853
854	/**
855	* Does global VT-x termination (called during module termination).
856	*/
857	VMMR0DECL(void) VMXR0GlobalTerm()
858	{
859	/* Nothing to do currently. */
860	}
861
862
863	/**
864	* Sets up and activates VT-x on the current CPU.
865	*
866	* @returns VBox status code.
867	* @param pCpu Pointer to the global CPU info struct.
868	* @param pVM Pointer to the VM (can be NULL after a host resume
869	* operation).
870	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
871	* fEnabledByHost is true).
872	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
873	* @a fEnabledByHost is true).
874	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
875	* enable VT-x/AMD-V on the host.
876	*/
877	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
878	{
879	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
880	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
881
882	if (!fEnabledByHost)
883	{
884	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
885	if (RT_FAILURE(rc))
886	return rc;
887	}
888
889	/*
890	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
891	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
892	* each time while reusing a VPID after hitting the MaxASID limit once.
893	*/
894	if ( pVM
895	&& pVM->hm.s.vmx.fVpid
896	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
897	{
898	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
899	pCpu->fFlushAsidBeforeUse = false;
900	}
901	else
902	pCpu->fFlushAsidBeforeUse = true;
903
904	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
905	++pCpu->cTlbFlushes;
906
907	return VINF_SUCCESS;
908	}
909
910
911	/**
912	* Deactivates VT-x on the current CPU.
913	*
914	* @returns VBox status code.
915	* @param pCpu Pointer to the global CPU info struct.
916	* @param pvCpuPage Pointer to the VMXON region.
917	* @param HCPhysCpuPage Physical address of the VMXON region.
918	*/
919	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
920	{
921	NOREF(pCpu);
922	NOREF(pvCpuPage);
923	NOREF(HCPhysCpuPage);
924
925	hmR0VmxLeaveRootMode();
926	return VINF_SUCCESS;
927	}
928
929
930	/**
931	* Sets the permission bits for the specified MSR in the MSR bitmap.
932	*
933	* @param pVCpu Pointer to the VMCPU.
934	* @param ulMSR The MSR value.
935	* @param enmRead Whether reading this MSR causes a VM-exit.
936	* @param enmWrite Whether writing this MSR causes a VM-exit.
937	*/
938	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, unsigned ulMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
939	{
940	unsigned ulBit;
941	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
942
943	/*
944	* Layout:
945	* 0x000 - 0x3ff - Low MSR read bits
946	* 0x400 - 0x7ff - High MSR read bits
947	* 0x800 - 0xbff - Low MSR write bits
948	* 0xc00 - 0xfff - High MSR write bits
949	*/
950	if (ulMsr <= 0x00001FFF)
951	{
952	/* Pentium-compatible MSRs */
953	ulBit = ulMsr;
954	}
955	else if ( ulMsr >= 0xC0000000
956	&& ulMsr <= 0xC0001FFF)
957	{
958	/* AMD Sixth Generation x86 Processor MSRs */
959	ulBit = (ulMsr - 0xC0000000);
960	pbMsrBitmap += 0x400;
961	}
962	else
963	{
964	AssertMsgFailed(("Invalid MSR %lx\n", ulMsr));
965	return;
966	}
967
968	Assert(ulBit <= 0x1fff);
969	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
970	ASMBitSet(pbMsrBitmap, ulBit);
971	else
972	ASMBitClear(pbMsrBitmap, ulBit);
973
974	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
975	ASMBitSet(pbMsrBitmap + 0x800, ulBit);
976	else
977	ASMBitClear(pbMsrBitmap + 0x800, ulBit);
978	}
979
980
981	/**
982	* Flushes the TLB using EPT.
983	*
984	* @returns VBox status code.
985	* @param pVM Pointer to the VM.
986	* @param pVCpu Pointer to the VMCPU.
987	* @param enmFlush Type of flush.
988	*/
989	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
990	{
991	AssertPtr(pVM);
992	Assert(pVM->hm.s.fNestedPaging);
993
994	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
995
996	uint64_t descriptor[2];
997	descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
998	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
999
1000	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1001	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
1002	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1003	}
1004
1005
1006	/**
1007	* Flushes the TLB using VPID.
1008	*
1009	* @returns VBox status code.
1010	* @param pVM Pointer to the VM.
1011	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1012	* enmFlush).
1013	* @param enmFlush Type of flush.
1014	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1015	* on @a enmFlush).
1016	*/
1017	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1018	{
1019	AssertPtr(pVM);
1020	Assert(pVM->hm.s.vmx.fVpid);
1021
1022	uint64_t descriptor[2];
1023	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1024	{
1025	descriptor[0] = 0;
1026	descriptor[1] = 0;
1027	}
1028	else
1029	{
1030	AssertPtr(pVCpu);
1031	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1032	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1033	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1034	descriptor[1] = GCPtr;
1035	}
1036
1037	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1038	AssertMsg(rc == VINF_SUCCESS,
1039	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1040	if ( RT_SUCCESS(rc)
1041	&& pVCpu)
1042	{
1043	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1044	}
1045	}
1046
1047
1048	/**
1049	* Invalidates a guest page by guest virtual address. Only relevant for
1050	* EPT/VPID, otherwise there is nothing really to invalidate.
1051	*
1052	* @returns VBox status code.
1053	* @param pVM Pointer to the VM.
1054	* @param pVCpu Pointer to the VMCPU.
1055	* @param GCVirt Guest virtual address of the page to invalidate.
1056	*/
1057	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1058	{
1059	AssertPtr(pVM);
1060	AssertPtr(pVCpu);
1061	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1062
1063	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1064	if (!fFlushPending)
1065	{
1066	/*
1067	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1068	* See @bugref{6043} and @bugref{6177}.
1069	*
1070	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1071	* function maybe called in a loop with individual addresses.
1072	*/
1073	if (pVM->hm.s.vmx.fVpid)
1074	{
1075	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1076	{
1077	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1078	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1079	}
1080	else
1081	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1082	}
1083	else if (pVM->hm.s.fNestedPaging)
1084	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1085	}
1086
1087	return VINF_SUCCESS;
1088	}
1089
1090
1091	/**
1092	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1093	* otherwise there is nothing really to invalidate.
1094	*
1095	* @returns VBox status code.
1096	* @param pVM Pointer to the VM.
1097	* @param pVCpu Pointer to the VMCPU.
1098	* @param GCPhys Guest physical address of the page to invalidate.
1099	*/
1100	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1101	{
1102	LogFlowFunc(("%RGp\n", GCPhys));
1103
1104	/*
1105	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1106	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1107	* This function might be called in a loop.
1108	*/
1109	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1110	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1111	return VINF_SUCCESS;
1112	}
1113
1114
1115	/**
1116	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1117	* case where neither EPT nor VPID is supported by the CPU.
1118	*
1119	* @param pVM Pointer to the VM.
1120	* @param pVCpu Pointer to the VMCPU.
1121	*
1122	* @remarks Called with interrupts disabled.
1123	*/
1124	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1125	{
1126	NOREF(pVM);
1127	AssertPtr(pVCpu);
1128	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1129	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1130
1131	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1132	AssertPtr(pCpu);
1133
1134	pVCpu->hm.s.TlbShootdown.cPages = 0;
1135	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1136	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1137	pVCpu->hm.s.fForceTLBFlush = false;
1138	return;
1139	}
1140
1141
1142	/**
1143	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1144	*
1145	* @param pVM Pointer to the VM.
1146	* @param pVCpu Pointer to the VMCPU.
1147	* @remarks All references to "ASID" in this function pertains to "VPID" in
1148	* Intel's nomenclature. The reason is, to avoid confusion in compare
1149	* statements since the host-CPU copies are named "ASID".
1150	*
1151	* @remarks Called with interrupts disabled.
1152	*/
1153	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1154	{
1155	AssertPtr(pVM);
1156	AssertPtr(pVCpu);
1157	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1158	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1159	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1160
1161	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1162	AssertPtr(pCpu);
1163
1164	/*
1165	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1166	* This can happen both for start & resume due to long jumps back to ring-3.
1167	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1168	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1169	*/
1170	bool fNewASID = false;
1171	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1172	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1173	{
1174	pVCpu->hm.s.fForceTLBFlush = true;
1175	fNewASID = true;
1176	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1177	}
1178
1179	/*
1180	* Check for explicit TLB shootdowns.
1181	*/
1182	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1183	{
1184	pVCpu->hm.s.fForceTLBFlush = true;
1185	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1186	}
1187
1188	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1189	if (pVCpu->hm.s.fForceTLBFlush)
1190	{
1191	if (fNewASID)
1192	{
1193	++pCpu->uCurrentAsid;
1194	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1195	{
1196	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1197	pCpu->cTlbFlushes++;
1198	pCpu->fFlushAsidBeforeUse = true;
1199	}
1200
1201	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1202	if (pCpu->fFlushAsidBeforeUse)
1203	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1204	}
1205	else
1206	{
1207	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1208	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1209	else
1210	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1211	}
1212
1213	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1214	pVCpu->hm.s.fForceTLBFlush = false;
1215	}
1216	else
1217	{
1218	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1219	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1220	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1221	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1222
1223	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1224	* not be executed. See hmQueueInvlPage() where it is commented
1225	* out. Support individual entry flushing someday. */
1226	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1227	{
1228	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1229
1230	/*
1231	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1232	* as supported by the CPU.
1233	*/
1234	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1235	{
1236	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1237	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1238	}
1239	else
1240	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1241	}
1242	else
1243	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1244	}
1245	pVCpu->hm.s.TlbShootdown.cPages = 0;
1246	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1247
1248	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1249	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1250	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1251	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1252	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1253	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1254
1255	/* Update VMCS with the VPID. */
1256	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1257	AssertRC(rc);
1258	}
1259
1260
1261	/**
1262	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1263	*
1264	* @returns VBox status code.
1265	* @param pVM Pointer to the VM.
1266	* @param pVCpu Pointer to the VMCPU.
1267	*
1268	* @remarks Called with interrupts disabled.
1269	*/
1270	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1271	{
1272	AssertPtr(pVM);
1273	AssertPtr(pVCpu);
1274	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1275	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1276
1277	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1278	AssertPtr(pCpu);
1279
1280	/*
1281	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1282	* This can happen both for start & resume due to long jumps back to ring-3.
1283	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1284	*/
1285	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1286	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1287	{
1288	pVCpu->hm.s.fForceTLBFlush = true;
1289	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1290	}
1291
1292	/* Check for explicit TLB shootdown flushes. */
1293	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1294	{
1295	pVCpu->hm.s.fForceTLBFlush = true;
1296	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1297	}
1298
1299	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1300	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1301
1302	if (pVCpu->hm.s.fForceTLBFlush)
1303	{
1304	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1305	pVCpu->hm.s.fForceTLBFlush = false;
1306	}
1307	else
1308	{
1309	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1310	* not be executed. See hmQueueInvlPage() where it is commented
1311	* out. Support individual entry flushing someday. */
1312	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1313	{
1314	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1315	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1316	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1317	}
1318	else
1319	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1320	}
1321
1322	pVCpu->hm.s.TlbShootdown.cPages= 0;
1323	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1324	}
1325
1326
1327	/**
1328	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1329	*
1330	* @returns VBox status code.
1331	* @param pVM Pointer to the VM.
1332	* @param pVCpu Pointer to the VMCPU.
1333	*
1334	* @remarks Called with interrupts disabled.
1335	*/
1336	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1337	{
1338	AssertPtr(pVM);
1339	AssertPtr(pVCpu);
1340	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1341	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1342
1343	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1344
1345	/*
1346	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1347	* This can happen both for start & resume due to long jumps back to ring-3.
1348	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1349	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1350	*/
1351	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1352	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1353	{
1354	pVCpu->hm.s.fForceTLBFlush = true;
1355	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1356	}
1357
1358	/* Check for explicit TLB shootdown flushes. */
1359	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1360	{
1361	/*
1362	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1363	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1364	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1365	*/
1366	pVCpu->hm.s.fForceTLBFlush = true;
1367	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1368	}
1369
1370	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1371	if (pVCpu->hm.s.fForceTLBFlush)
1372	{
1373	++pCpu->uCurrentAsid;
1374	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1375	{
1376	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1377	pCpu->fFlushAsidBeforeUse = true;
1378	pCpu->cTlbFlushes++;
1379	}
1380
1381	pVCpu->hm.s.fForceTLBFlush = false;
1382	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1383	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1384	if (pCpu->fFlushAsidBeforeUse)
1385	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1386	}
1387	else
1388	{
1389	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1390	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1391	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1392	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1393
1394	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1395	* not be executed. See hmQueueInvlPage() where it is commented
1396	* out. Support individual entry flushing someday. */
1397	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1398	{
1399	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1400	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1401	{
1402	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1403	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1404	}
1405	else
1406	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1407	}
1408	else
1409	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1410	}
1411
1412	pVCpu->hm.s.TlbShootdown.cPages = 0;
1413	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1414
1415	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1416	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1417	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1418	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1419	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1420	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1421
1422	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1423	AssertRC(rc);
1424	}
1425
1426
1427	/**
1428	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1429	* TLB entries from the host TLB before VM-entry.
1430	*
1431	* @returns VBox status code.
1432	* @param pVM Pointer to the VM.
1433	*/
1434	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1435	{
1436	/*
1437	* Determine optimal flush type for nested paging.
1438	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1439	* guest execution (see hmR3InitFinalizeR0()).
1440	*/
1441	if (pVM->hm.s.fNestedPaging)
1442	{
1443	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1444	{
1445	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1446	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1447	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1448	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1449	else
1450	{
1451	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1452	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1453	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1454	}
1455
1456	/* Make sure the write-back cacheable memory type for EPT is supported. */
1457	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1458	{
1459	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1460	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1461	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1462	}
1463	}
1464	else
1465	{
1466	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1467	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1468	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1469	}
1470	}
1471
1472	/*
1473	* Determine optimal flush type for VPID.
1474	*/
1475	if (pVM->hm.s.vmx.fVpid)
1476	{
1477	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1478	{
1479	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1480	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1481	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1482	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1483	else
1484	{
1485	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1486	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1487	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1488	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1489	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1490	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1491	pVM->hm.s.vmx.fVpid = false;
1492	}
1493	}
1494	else
1495	{
1496	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1497	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1498	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1499	pVM->hm.s.vmx.fVpid = false;
1500	}
1501	}
1502
1503	/*
1504	* Setup the handler for flushing tagged-TLBs.
1505	*/
1506	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1507	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbBoth;
1508	else if (pVM->hm.s.fNestedPaging)
1509	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbEpt;
1510	else if (pVM->hm.s.vmx.fVpid)
1511	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbVpid;
1512	else
1513	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbNone;
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	/**
1519	* Sets up pin-based VM-execution controls in the VMCS.
1520	*
1521	* @returns VBox status code.
1522	* @param pVM Pointer to the VM.
1523	* @param pVCpu Pointer to the VMCPU.
1524	*/
1525	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1526	{
1527	AssertPtr(pVM);
1528	AssertPtr(pVCpu);
1529
1530	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1531	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1532
1533	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1534	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1535	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1536
1537	/* Enable the VMX preemption timer. */
1538	if (pVM->hm.s.vmx.fUsePreemptTimer)
1539	{
1540	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1541	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1542	}
1543
1544	if ((val & zap) != val)
1545	{
1546	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1547	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1548	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1549	}
1550
1551	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1552	AssertRCReturn(rc, rc);
1553
1554	/* Update VCPU with the currently set pin-based VM-execution controls. */
1555	pVCpu->hm.s.vmx.u32PinCtls = val;
1556	return rc;
1557	}
1558
1559
1560	/**
1561	* Sets up processor-based VM-execution controls in the VMCS.
1562	*
1563	* @returns VBox status code.
1564	* @param pVM Pointer to the VM.
1565	* @param pVMCPU Pointer to the VMCPU.
1566	*/
1567	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1568	{
1569	AssertPtr(pVM);
1570	AssertPtr(pVCpu);
1571
1572	int rc = VERR_INTERNAL_ERROR_5;
1573	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1574	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1575
1576	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1577	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1578	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1579	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1580	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1581	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1582	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1583
1584	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1585	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1586	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1587	{
1588	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1589	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1590	}
1591
1592	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1593	if (!pVM->hm.s.fNestedPaging)
1594	{
1595	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1596	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1597	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1598	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1599	}
1600
1601	/* Use TPR shadowing if supported by the CPU. */
1602	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1603	{
1604	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1605	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1606	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1607	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1608	AssertRCReturn(rc, rc);
1609
1610	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1611	/* CR8 writes causes a VM-exit based on TPR threshold. */
1612	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1613	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1614	}
1615	else
1616	{
1617	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1618	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1619	}
1620
1621	/* Use MSR-bitmaps if supported by the CPU. */
1622	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1623	{
1624	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1625
1626	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1627	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1628	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1629	AssertRCReturn(rc, rc);
1630
1631	/*
1632	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1633	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1634	*/
1635	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1636	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1637	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1638	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1639	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1640	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1641	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1642	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1643	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1644	}
1645
1646	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1647	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1648	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1649
1650	if ((val & zap) != val)
1651	{
1652	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1653	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1654	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1655	}
1656
1657	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1658	AssertRCReturn(rc, rc);
1659
1660	/* Update VCPU with the currently set processor-based VM-execution controls. */
1661	pVCpu->hm.s.vmx.u32ProcCtls = val;
1662
1663	/*
1664	* Secondary processor-based VM-execution controls.
1665	*/
1666	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1667	{
1668	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1669	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1670
1671	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1672	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1673
1674	if (pVM->hm.s.fNestedPaging)
1675	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1676	else
1677	{
1678	/*
1679	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1680	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1681	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1682	*/
1683	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1684	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1685	}
1686
1687	if (pVM->hm.s.vmx.fVpid)
1688	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1689
1690	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1691	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1692
1693	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1694	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1695	* done dynamically. */
1696	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1697	{
1698	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1699	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1700	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1701	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1702	AssertRCReturn(rc, rc);
1703	}
1704
1705	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1706	{
1707	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1708	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1709	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1710	}
1711
1712	if ((val & zap) != val)
1713	{
1714	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1715	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1716	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1717	}
1718
1719	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1720	AssertRCReturn(rc, rc);
1721
1722	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1723	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1724	}
1725
1726	return VINF_SUCCESS;
1727	}
1728
1729
1730	/**
1731	* Sets up miscellaneous (everything other than Pin & Processor-based
1732	* VM-execution) control fields in the VMCS.
1733	*
1734	* @returns VBox status code.
1735	* @param pVM Pointer to the VM.
1736	* @param pVCpu Pointer to the VMCPU.
1737	*/
1738	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1739	{
1740	AssertPtr(pVM);
1741	AssertPtr(pVCpu);
1742
1743	int rc = VERR_GENERAL_FAILURE;
1744
1745	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1746	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1747
1748	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1749
1750	/*
1751	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1752	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1753	* We thus use the exception bitmap to control it rather than use both.
1754	*/
1755	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1756	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1757
1758	/** @todo Explore possibility of using IO-bitmaps. */
1759	/* All IO & IOIO instructions cause VM-exits. */
1760	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1761	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1762
1763	/* Setup MSR autoloading/autostoring. */
1764	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1765	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1766	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1767	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1768	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1769	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1770
1771	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1772	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1773	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1774	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1775
1776	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1777	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1778
1779	/* Setup debug controls */
1780	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1781	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1782	AssertRCReturn(rc, rc);
1783	return rc;
1784	}
1785
1786
1787	/**
1788	* Sets up the initial exception bitmap in the VMCS based on static conditions
1789	* (i.e. conditions that cannot ever change at runtime).
1790	*
1791	* @returns VBox status code.
1792	* @param pVM Pointer to the VM.
1793	* @param pVCpu Pointer to the VMCPU.
1794	*/
1795	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1796	{
1797	AssertPtr(pVM);
1798	AssertPtr(pVCpu);
1799
1800	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1801
1802	uint32_t u32XcptBitmap = 0;
1803
1804	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1805	if (!pVM->hm.s.fNestedPaging)
1806	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1807
1808	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1809	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1810	AssertRCReturn(rc, rc);
1811	return rc;
1812	}
1813
1814
1815	/**
1816	* Sets up the initial guest-state mask. The guest-state mask is consulted
1817	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1818	* for the nested virtualization case (as it would cause a VM-exit).
1819	*
1820	* @param pVCpu Pointer to the VMCPU.
1821	*/
1822	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1823	{
1824	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1825	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1826	return VINF_SUCCESS;
1827	}
1828
1829
1830	/**
1831	* Does per-VM VT-x initialization.
1832	*
1833	* @returns VBox status code.
1834	* @param pVM Pointer to the VM.
1835	*/
1836	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1837	{
1838	LogFlowFunc(("pVM=%p\n", pVM));
1839
1840	int rc = hmR0VmxStructsAlloc(pVM);
1841	if (RT_FAILURE(rc))
1842	{
1843	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1844	return rc;
1845	}
1846
1847	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1848	{
1849	PVMCPU pVCpu = &pVM->aCpus[i];
1850
1851	/* Current guest paging mode. */
1852	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1853	}
1854
1855	return VINF_SUCCESS;
1856	}
1857
1858
1859	/**
1860	* Does per-VM VT-x termination.
1861	*
1862	* @returns VBox status code.
1863	* @param pVM Pointer to the VM.
1864	*/
1865	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1866	{
1867	LogFlowFunc(("pVM=%p\n", pVM));
1868
1869	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1870	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1871	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1872	#endif
1873	hmR0VmxStructsFree(pVM);
1874	return VINF_SUCCESS;
1875	}
1876
1877
1878	/**
1879	* Sets up the VM for execution under VT-x.
1880	* This function is only called once per-VM during initalization.
1881	*
1882	* @returns VBox status code.
1883	* @param pVM Pointer to the VM.
1884	*/
1885	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1886	{
1887	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1888	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1889
1890	LogFlowFunc(("pVM=%p\n", pVM));
1891
1892	/*
1893	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1894	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1895	*/
1896	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1897	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1898	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1899	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1900	{
1901	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1902	return VERR_INTERNAL_ERROR;
1903	}
1904
1905	/* Initialize these always, see hmR3InitFinalizeR0().*/
1906	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1907	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1908
1909	/* Setup the tagged-TLB flush handlers. */
1910	int rc = hmR0VmxSetupTaggedTlb(pVM);
1911	if (RT_FAILURE(rc))
1912	{
1913	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1914	return rc;
1915	}
1916
1917	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1918	{
1919	PVMCPU pVCpu = &pVM->aCpus[i];
1920	AssertPtr(pVCpu);
1921	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1922
1923	/* Set revision dword at the beginning of the VMCS structure. */
1924	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1925
1926	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1927	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1928	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1929	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1930
1931	/* Load this VMCS as the current VMCS. */
1932	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1933	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1934	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1935
1936	/* Setup the pin-based VM-execution controls. */
1937	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1938	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1939	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1940
1941	/* Setup the processor-based VM-execution controls. */
1942	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1943	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1944	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1945
1946	/* Setup the rest (miscellaneous) VM-execution controls. */
1947	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1948	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1949	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1950
1951	/* Setup the initial exception bitmap. */
1952	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1953	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1954	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1955
1956	/* Setup the initial guest-state mask. */
1957	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1958	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1959	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1960
1961	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1962	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1963	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1964	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1965	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1966	#endif
1967
1968	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
1969	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1970	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1971	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1972
1973	/* Update the last error record for this VCPU. */
1974	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
1975	}
1976
1977	return VINF_SUCCESS;
1978	}
1979
1980
1981	/**
1982	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
1983	* the VMCS.
1984	*
1985	* @returns VBox status code.
1986	* @param pVM Pointer to the VM.
1987	* @param pVCpu Pointer to the VMCPU.
1988	*/
1989	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
1990	{
1991	RTCCUINTREG uReg = ASMGetCR0();
1992	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
1993
1994	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1995	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
1996	if (VMX_IS_64BIT_HOST_MODE())
1997	{
1998	uint64_t uReg = hmR0Get64bitCR3();
1999	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2000	}
2001	else
2002	#endif
2003	{
2004	uReg = ASMGetCR3();
2005	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2006	}
2007
2008	uReg = ASMGetCR4();
2009	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2010	AssertRCReturn(rc, rc);
2011	return rc;
2012	}
2013
2014
2015	/**
2016	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2017	* the host-state area in the VMCS.
2018	*
2019	* @returns VBox status code.
2020	* @param pVM Pointer to the VM.
2021	* @param pVCpu Pointer to the VMCPU.
2022	*/
2023	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2024	{
2025	int rc = VERR_INTERNAL_ERROR_5;
2026	RTSEL uSelCS = 0;
2027	RTSEL uSelSS = 0;
2028	RTSEL uSelDS = 0;
2029	RTSEL uSelES = 0;
2030	RTSEL uSelFS = 0;
2031	RTSEL uSelGS = 0;
2032	RTSEL uSelTR = 0;
2033
2034	/*
2035	* Host Selector registers.
2036	*/
2037	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2038	if (VMX_IS_64BIT_HOST_MODE())
2039	{
2040	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2041	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2042	}
2043	else
2044	{
2045	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2046	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2047	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2048	}
2049	#else
2050	uSelCS = ASMGetCS();
2051	uSelSS = ASMGetSS();
2052	#endif
2053
2054	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2055	/** @todo Verify if we have any platform that actually run with DS or ES with
2056	* RPL != 0 in kernel space. */
2057	uSelDS = 0;
2058	uSelES = 0;
2059	uSelFS = 0;
2060	uSelGS = 0;
2061	uSelTR = ASMGetTR();
2062
2063	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2064	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2065	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2066	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2067	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2068	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2069	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2070	Assert(uSelCS != 0);
2071	Assert(uSelTR != 0);
2072
2073	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2074	#if 0
2075	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2076	Assert(uSelSS != 0);
2077	#endif
2078
2079	/* Write these host selector fields into the host-state area in the VMCS. */
2080	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2081	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2082	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2083	#if 0
2084	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2085	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2086	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2087	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2088	#endif
2089	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2090	AssertRCReturn(rc, rc);
2091
2092	/*
2093	* Host GDTR and IDTR.
2094	*/
2095	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2096	* be safe to -not- save and restore GDTR and IDTR in the assembly
2097	* code and just do it here and don't care if the limits are zapped on
2098	* VM-exit. */
2099	RTGDTR Gdtr;
2100	RT_ZERO(Gdtr);
2101	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2102	if (VMX_IS_64BIT_HOST_MODE())
2103	{
2104	X86XDTR64 Gtr64;
2105	X86XDTR64 Idtr64;
2106	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2107	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2108	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2109	Gdtr.cbGdt = Gdtr64.cb;
2110	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2111	}
2112	else
2113	#endif
2114	{
2115	RTIDTR Idtr;
2116	ASMGetGDTR(&Gdtr);
2117	ASMGetIDTR(&Idtr);
2118	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2119	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2120	}
2121	AssertRCReturn(rc, rc);
2122
2123	/*
2124	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2125	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2126	*/
2127	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2128	{
2129	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2130	return VERR_VMX_INVALID_HOST_STATE;
2131	}
2132
2133	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2134	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2135	if (VMX_IS_64BIT_HOST_MODE())
2136	{
2137	/* We need the 64-bit TR base for hybrid darwin. */
2138	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2139	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2140	}
2141	else
2142	#endif
2143	{
2144	uintptr_t uTRBase;
2145	#if HC_ARCH_BITS == 64
2146	uTRBase = X86DESC64_BASE(pDesc);
2147	#else
2148	uTRBase = X86DESC_BASE(pDesc);
2149	#endif
2150	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2151	}
2152	AssertRCReturn(rc, rc);
2153
2154	/*
2155	* Host FS base and GS base.
2156	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2157	* would take care of the bases. In 64-bit, the MSRs come into play.
2158	*/
2159	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2160	if (VMX_IS_64BIT_HOST_MODE())
2161	{
2162	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2163	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2164	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2165	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2166	AssertRCReturn(rc, rc);
2167	}
2168	#endif
2169	return rc;
2170	}
2171
2172
2173	/**
2174	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2175	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2176	* the host after every successful VM exit.
2177	*
2178	* @returns VBox status code.
2179	* @param pVM Pointer to the VM.
2180	* @param pVCpu Pointer to the VMCPU.
2181	*/
2182	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2183	{
2184	AssertPtr(pVCpu);
2185	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2186
2187	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2188	unsigned idxHostMsr = 0;
2189	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2190
2191	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2192	{
2193	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2194	pHostMsr->u32Reserved = 0;
2195	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2196	if (CPUMIsGuestInLongMode(pVCpu))
2197	{
2198	/* Must match the EFER value in our 64 bits switcher. */
2199	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2200	}
2201	else
2202	#endif
2203	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2204	pHostMsr++; idxHostMsr++;
2205	}
2206
2207	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2208	if (VMX_IS_64BIT_HOST_MODE())
2209	{
2210	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2211	pHostMsr->u32Reserved = 0;
2212	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2213	pHostMsr++; idxHostMsr++;
2214	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2215	pHostMsr->u32Reserved = 0;
2216	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2217	pHostMsr++; idxHostMsr++;
2218	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2219	pHostMsr->u32Reserved = 0;
2220	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2221	pHostMsr++; idxHostMsr++;
2222	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2223	#if 0
2224	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2225	pMsr->u32Reserved = 0;
2226	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2227	pHostMsr++; idxHostMsr++;
2228	#endif
2229	}
2230	#endif
2231
2232	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2233	if (RT_UNLIKELY(idxHostMsr > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2234	{
2235	LogRel(("idxHostMsr=%u Cpu=%u\n", idxHostMsr, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2236	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2237	}
2238
2239	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxHostMsr);
2240
2241	/*
2242	* Host Sysenter MSRs.
2243	*/
2244	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2245	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2246	if (VMX_IS_64BIT_HOST_MODE())
2247	{
2248	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2249	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2250	}
2251	else
2252	{
2253	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2254	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2255	}
2256	#elif HC_ARCH_BITS == 32
2257	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2258	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2259	#else
2260	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2261	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2262	#endif
2263	AssertRCReturn(rc, rc);
2264
2265	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2266	* hmR0VmxSetupExitCtls() !! */
2267	return rc;
2268	}
2269
2270
2271	/**
2272	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2273	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2274	* controls".
2275	*
2276	* @returns VBox status code.
2277	* @param pVM Pointer to the VM.
2278	* @param pVCpu Pointer to the VMCPU.
2279	* @param pCtx Pointer to the guest-CPU context.
2280	*
2281	* @remarks No-long-jump zone!!!
2282	*/
2283	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2284	{
2285	int rc = VINF_SUCCESS;
2286	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2287	{
2288	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2289	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2290
2291	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2292	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2293
2294	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2295	if (CPUMIsGuestInLongModeEx(pCtx))
2296	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2297	else
2298	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2299
2300	/*
2301	* The following should not be set (since we're not in SMM mode):
2302	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2303	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2304	*/
2305
2306	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2307	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2308	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2309
2310	if ((val & zap) != val)
2311	{
2312	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2313	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2314	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2315	}
2316
2317	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2318	AssertRCReturn(rc, rc);
2319
2320	/* Update VCPU with the currently set VM-exit controls. */
2321	pVCpu->hm.s.vmx.u32EntryCtls = val;
2322	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2323	}
2324	return rc;
2325	}
2326
2327
2328	/**
2329	* Sets up the VM-exit controls in the VMCS.
2330	*
2331	* @returns VBox status code.
2332	* @param pVM Pointer to the VM.
2333	* @param pVCpu Pointer to the VMCPU.
2334	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2335	* out-of-sync. Make sure to update the required fields
2336	* before using them.
2337	*
2338	* @remarks requires EFER.
2339	*/
2340	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2341	{
2342	int rc = VINF_SUCCESS;
2343	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2344	{
2345	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2346	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2347
2348	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2349	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2350
2351	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2352	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2353	if (VMX_IS_64BIT_HOST_MODE())
2354	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2355	else
2356	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2357	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2358	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2359	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2360	else
2361	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2362	#endif
2363
2364	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2365	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2366
2367	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2368	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2369	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2370	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2371	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2372
2373	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2374	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2375
2376	if ((val & zap) != val)
2377	{
2378	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2379	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2380	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2381	}
2382
2383	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2384	AssertRCReturn(rc, rc);
2385
2386	/* Update VCPU with the currently set VM-exit controls. */
2387	pVCpu->hm.s.vmx.u32ExitCtls = val;
2388	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2389	}
2390	return rc;
2391	}
2392
2393
2394	/**
2395	* Loads the guest APIC and related state.
2396	*
2397	* @returns VBox status code.
2398	* @param pVM Pointer to the VM.
2399	* @param pVCpu Pointer to the VMCPU.
2400	* @param pCtx Pointer to the guest-CPU context.
2401	*/
2402	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2403	{
2404	int rc = VINF_SUCCESS;
2405	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2406	{
2407	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2408	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2409	{
2410	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2411
2412	bool fPendingIntr = false;
2413	uint8_t u8GuestTpr = 0;
2414	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2415	AssertRCReturn(rc, rc);
2416
2417	/*
2418	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2419	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2420	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2421	* the interrupt when we VM-exit for other reasons.
2422	*/
2423	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2424	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2425	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2426	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2427
2428	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2429	AssertRCReturn(rc, rc);
2430
2431	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2432	if (pVM->hm.s.fTPRPatchingActive)
2433	{
2434	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2435	pCtx->msrLSTAR = u8GuestTpr;
2436	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2437	{
2438	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2439	if (fPendingIntr)
2440	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2441	else
2442	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443	}
2444	}
2445	}
2446
2447	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2448	}
2449	return rc;
2450	}
2451
2452
2453	/**
2454	* Loads the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
2455	* into the guest-state area in the VMCS.
2456	*
2457	* @param pVM Pointer to the VM.
2458	* @param pVCpu Pointer to the VMCPU.
2459	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2460	* out-of-sync. Make sure to update the required fields
2461	* before using them.
2462	*
2463	* @remarks No-long-jump zone!!!
2464	*/
2465	DECLINLINE(void) hmR0VmxLoadGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2466	{
2467	/*
2468	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2469	* inhibit interrupts or clear any existing interrupt-inhibition.
2470	*/
2471	uint32_t uIntrState = 0;
2472	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2473	{
2474	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2475	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS)),
2476	("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2477	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2478	{
2479	/*
2480	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2481	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2482	* hmR0VmxInjectPendingInterrupt() relies on us clearing this flag here.
2483	*/
2484	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2485	}
2486	else if (pMixedCtx->eflags.Bits.u1IF)
2487	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2488	else
2489	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2490	}
2491
2492	Assert(!(uIntrState & 0xfffffff0)); /* Bits 31:4 MBZ. */
2493	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2494	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2495	AssertRC(rc);
2496	}
2497
2498
2499	/**
2500	* Loads the guest's RIP into the guest-state area in the VMCS.
2501	*
2502	* @returns VBox status code.
2503	* @param pVM Pointer to the VM.
2504	* @param pVCpu Pointer to the VMCPU.
2505	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2506	* out-of-sync. Make sure to update the required fields
2507	* before using them.
2508	*
2509	* @remarks No-long-jump zone!!!
2510	*/
2511	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2512	{
2513	int rc = VINF_SUCCESS;
2514	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2515	{
2516	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2517	AssertRCReturn(rc, rc);
2518	Log(("VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2519	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2520	}
2521	return rc;
2522	}
2523
2524
2525	/**
2526	* Loads the guest's RSP into the guest-state area in the VMCS.
2527	*
2528	* @returns VBox status code.
2529	* @param pVM Pointer to the VM.
2530	* @param pVCpu Pointer to the VMCPU.
2531	* @param pCtx Pointer to the guest-CPU context.
2532	*
2533	* @remarks No-long-jump zone!!!
2534	*/
2535	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2536	{
2537	int rc = VINF_SUCCESS;
2538	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2539	{
2540	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2541	AssertRCReturn(rc, rc);
2542	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2543	}
2544	return rc;
2545	}
2546
2547
2548	/**
2549	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2550	*
2551	* @returns VBox status code.
2552	* @param pVM Pointer to the VM.
2553	* @param pVCpu Pointer to the VMCPU.
2554	* @param pCtx Pointer to the guest-CPU context.
2555	*
2556	* @remarks No-long-jump zone!!!
2557	*/
2558	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2559	{
2560	int rc = VINF_SUCCESS;
2561	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2562	{
2563	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2564	Let us assert it as such and use native-width VMWRITE. */
2565	X86RFLAGS uRFlags = pCtx->rflags;
2566	Assert(uRFlags.u64 >> 32 == 0);
2567	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2568	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2569
2570	/*
2571	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2572	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2573	*/
2574	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2575	{
2576	Assert(pVM->hm.s.vmx.pRealModeTSS);
2577	Assert(PDMVmmDevHeapIsEnabled(pVM));
2578	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2579	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2580	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2581	}
2582
2583	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2584	AssertRCReturn(rc, rc);
2585
2586	Log(("VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2587	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2588	}
2589	return rc;
2590	}
2591
2592
2593	/**
2594	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2595	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2596	* assembly code.
2597	*
2598	* @returns VBox status code.
2599	* @param pVM Pointer to the VM.
2600	* @param pVCpu Pointer to the VMCPU.
2601	* @param pCtx Pointer to the guest-CPU context.
2602	*
2603	* @remarks No-long-jump zone!!!
2604	*/
2605	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2606	{
2607	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2608	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2609	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2610	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2611	return rc;
2612	}
2613
2614
2615	/**
2616	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2617	* in the VMCS.
2618	*
2619	* @returns VBox status code.
2620	* @param pVM Pointer to the VM.
2621	* @param pVCpu Pointer to the VMCPU.
2622	* @param pCtx Pointer to the guest-CPU context.
2623	*
2624	* @remarks No-long-jump zone!!!
2625	*/
2626	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2627	{
2628	int rc = VINF_SUCCESS;
2629
2630	/*
2631	* Guest CR0.
2632	* Guest FPU.
2633	*/
2634	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2635	{
2636	uint64_t u64GuestCR0 = pCtx->cr0;
2637
2638	/* The guest's view (read access) of its CR0 is unblemished. */
2639	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2640	AssertRCReturn(rc, rc);
2641	Log2(("VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2642
2643	/* Setup VT-x's view of the guest CR0. */
2644	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2645	if (pVM->hm.s.fNestedPaging)
2646	{
2647	if (CPUMIsGuestPagingEnabledEx(pCtx))
2648	{
2649	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2650	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2651	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2652	}
2653	else
2654	{
2655	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2656	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2657	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2658	}
2659
2660	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2661	AssertRCReturn(rc, rc);
2662	}
2663	else
2664	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2665
2666	/*
2667	* Guest FPU bits.
2668	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2669	* CPUs to support VT-x (prob. means all the way up to Nehalem) and no mention of with regards to UX in VM-entry checks.
2670	*/
2671	u64GuestCR0 \|= X86_CR0_NE;
2672	bool fInterceptNM = false;
2673	if (CPUMIsGuestFPUStateActive(pVCpu))
2674	{
2675	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2676	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2677	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2678	}
2679	else
2680	{
2681	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2682	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2683	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2684	}
2685
2686	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2687	bool fInterceptMF = false;
2688	if (!(pCtx->cr0 & X86_CR0_NE))
2689	fInterceptMF = true;
2690
2691	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2692	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2693	{
2694	Assert(PDMVmmDevHeapIsEnabled(pVM));
2695	Assert(pVM->hm.s.vmx.pRealModeTSS);
2696	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2697	fInterceptNM = true;
2698	fInterceptMF = true;
2699	}
2700	else
2701	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2702
2703	if (fInterceptNM)
2704	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2705	else
2706	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2707
2708	if (fInterceptMF)
2709	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2710	else
2711	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2712
2713	/* Additional intercepts for debugging, define these yourself explicitly. */
2714	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2715	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2716	\| RT_BIT(X86_XCPT_DB)
2717	\| RT_BIT(X86_XCPT_DE)
2718	\| RT_BIT(X86_XCPT_NM)
2719	\| RT_BIT(X86_XCPT_UD)
2720	\| RT_BIT(X86_XCPT_NP)
2721	\| RT_BIT(X86_XCPT_SS)
2722	\| RT_BIT(X86_XCPT_GP)
2723	\| RT_BIT(X86_XCPT_PF)
2724	\| RT_BIT(X86_XCPT_MF);
2725	#elif defined(VBOX_ALWAYS_TRAP_PF)
2726	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2727	#endif
2728
2729	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2730
2731	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2732	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2733	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2734	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2735	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2736	else
2737	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2738
2739	u64GuestCR0 \|= uSetCR0;
2740	u64GuestCR0 &= uZapCR0;
2741	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2742
2743	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2744	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2745	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2746	Log2(("VMX_VMCS_GUEST_CR0=%#RX32\n", (uint32_t)u64GuestCR0));
2747
2748	/*
2749	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2750	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2751	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2752	*/
2753	uint64_t u64CR0Mask = 0;
2754	u64CR0Mask = X86_CR0_PE
2755	\| X86_CR0_WP
2756	\| X86_CR0_PG
2757	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2758	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2759	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2760
2761	/* We don't need to intercept changes to CR0.PE with unrestricted guests. */
2762	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2763	u64CR0Mask &= ~X86_CR0_PE;
2764
2765	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2766	if (fInterceptNM)
2767	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2768	else
2769	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2770
2771	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2772	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2773	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2774	AssertRCReturn(rc, rc);
2775
2776	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2777	}
2778
2779	/*
2780	* Guest CR2.
2781	* It's always loaded in the assembler code. Nothing to do here.
2782	*/
2783
2784	/*
2785	* Guest CR3.
2786	*/
2787	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2788	{
2789	uint64_t u64GuestCR3 = 0;
2790	if (pVM->hm.s.fNestedPaging)
2791	{
2792	pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2793
2794	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2795	Assert(pVCpu->hm.s.vmx.GCPhysEPTP);
2796	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff0000000000000ULL));
2797	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2798
2799	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2800	pVCpu->hm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2801	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2802
2803	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2804	AssertMsg( ((pVCpu->hm.s.vmx.GCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2805	&& ((pVCpu->hm.s.vmx.GCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2806	("EPTP %#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2807
2808	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2809	AssertRCReturn(rc, rc);
2810	Log(("VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2811
2812	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2813	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2814	{
2815	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2816	if (CPUMIsGuestInPAEModeEx(pCtx))
2817	{
2818	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2819	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2820	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2821	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2822	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2823	AssertRCReturn(rc, rc);
2824	}
2825
2826	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2827	have Unrestricted Execution to handle the guest when it's not using paging. */
2828	u64GuestCR3 = pCtx->cr3;
2829	}
2830	else
2831	{
2832	/*
2833	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2834	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2835	* EPT takes care of translating it to host-physical addresses.
2836	*/
2837	RTGCPHYS GCPhys;
2838	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2839	Assert(PDMVmmDevHeapIsEnabled(pVM));
2840
2841	/* We obtain it here every time as the guest could have relocated this PCI region. */
2842	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2843	AssertRCReturn(rc, rc);
2844
2845	u64GuestCR3 = GCPhys;
2846	}
2847	}
2848	else
2849	{
2850	/* Non-nested paging case, just use the hypervisor's CR3. */
2851	u64GuestCR3 = PGMGetHyperCR3(pVCpu);
2852	}
2853
2854	Log2(("VMX_VMCS_GUEST_CR3=%#RX64\n", u64GuestCR3));
2855	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, u64GuestCR3);
2856	AssertRCReturn(rc, rc);
2857
2858	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2859	}
2860
2861	/*
2862	* Guest CR4.
2863	*/
2864	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2865	{
2866	uint64_t u64GuestCR4 = pCtx->cr4;
2867
2868	/* The guest's view of its CR4 is unblemished. */
2869	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2870	AssertRCReturn(rc, rc);
2871	Log2(("VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2872
2873	/* Setup VT-x's view of the guest CR4. */
2874	/*
2875	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2876	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2877	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2878	*/
2879	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2880	{
2881	Assert(pVM->hm.s.vmx.pRealModeTSS);
2882	Assert(PDMVmmDevHeapIsEnabled(pVM));
2883	u64GuestCR4 &= ~X86_CR4_VME;
2884	}
2885
2886	if (pVM->hm.s.fNestedPaging)
2887	{
2888	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2889	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2890	{
2891	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2892	u64GuestCR4 \|= X86_CR4_PSE;
2893	/* Our identity mapping is a 32 bits page directory. */
2894	u64GuestCR4 &= ~X86_CR4_PAE;
2895	}
2896	/* else use guest CR4.*/
2897	}
2898	else
2899	{
2900	/*
2901	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2902	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2903	*/
2904	switch (pVCpu->hm.s.enmShadowMode)
2905	{
2906	case PGMMODE_REAL: /* Real-mode. */
2907	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2908	case PGMMODE_32_BIT: /* 32-bit paging. */
2909	{
2910	u64GuestCR4 &= ~X86_CR4_PAE;
2911	break;
2912	}
2913
2914	case PGMMODE_PAE: /* PAE paging. */
2915	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2916	{
2917	u64GuestCR4 \|= X86_CR4_PAE;
2918	break;
2919	}
2920
2921	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2922	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2923	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2924	break;
2925	#endif
2926	default:
2927	AssertFailed();
2928	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2929	}
2930	}
2931
2932	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2933	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2934	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2935	u64GuestCR4 \|= uSetCR4;
2936	u64GuestCR4 &= uZapCR4;
2937
2938	/* Write VT-x's view of the guest CR4 into the VMCS. */
2939	Log2(("VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2940	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2941
2942	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2943	uint64_t u64CR4Mask = 0;
2944	u64CR4Mask = X86_CR4_VME
2945	\| X86_CR4_PAE
2946	\| X86_CR4_PGE
2947	\| X86_CR4_PSE
2948	\| X86_CR4_VMXE;
2949	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
2950	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
2951	AssertRCReturn(rc, rc);
2952
2953	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
2954	}
2955	return rc;
2956	}
2957
2958
2959	/**
2960	* Loads the guest debug registers into the guest-state area in the VMCS.
2961	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
2962	*
2963	* @returns VBox status code.
2964	* @param pVM Pointer to the VM.
2965	* @param pVCpu Pointer to the VMCPU.
2966	* @param pCtx Pointer to the guest-CPU context.
2967	*
2968	* @remarks No-long-jump zone!!!
2969	*/
2970	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2971	{
2972	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
2973	return VINF_SUCCESS;
2974
2975	#ifdef DEBUG
2976	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
2977	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
2978	{
2979	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
2980	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
2981	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
2982	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
2983	}
2984	#endif
2985
2986	int rc = VERR_INTERNAL_ERROR_5;
2987	bool fInterceptDB = false;
2988	bool fInterceptMovDRx = false;
2989	if (DBGFIsStepping(pVCpu))
2990	{
2991	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
2992	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2993	{
2994	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2995	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2996	AssertRCReturn(rc, rc);
2997	Assert(fInterceptDB == false);
2998	}
2999	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
3000	{
3001	Assert(fInterceptDB == false);
3002	/** @todo can we somehow signal DBGF that it cannot single-step instead of
3003	* just continuing? */
3004	}
3005	else
3006	fInterceptDB = true;
3007	}
3008	else
3009	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
3010
3011	/*
3012	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
3013	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
3014	* The same for the hypervisor DRx registers, priority is for the guest here.
3015	*/
3016	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3017	&& !CPUMIsGuestDebugStateActive(pVCpu))
3018	{
3019	/* Save the host and load the guest debug registers. This will make the guest debug state active. */
3020	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3021	AssertRC(rc);
3022	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3023	Assert(fInterceptMovDRx == false);
3024	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3025	}
3026	else if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)
3027	&& !CPUMIsHyperDebugStateActive(pVCpu))
3028	{
3029	/* Save the host and load the hypervisor debug registers. This will make the hyper debug state active. */
3030	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3031	AssertRC(rc);
3032	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3033	fInterceptMovDRx = true;
3034	}
3035	else
3036	Assert(fInterceptMovDRx == false); /* No need to intercept MOV DRx if DBGF is not active nor the guest is debugging. */
3037
3038	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3039	if (fInterceptDB)
3040	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3041	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3042	{
3043	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3044	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3045	#endif
3046	}
3047
3048	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3049	if (fInterceptMovDRx)
3050	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3051	else
3052	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3053
3054	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3055	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3056
3057	/* The guest's view of its DR7 is unblemished. */
3058	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3059
3060	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3061	return rc;
3062	}
3063
3064
3065	#ifdef DEBUG
3066	/**
3067	* Debug function to validate segment registers.
3068	*/
3069	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3070	{
3071	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3072	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3073	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3074	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3075	{
3076	/* Protected mode checks */
3077	/* CS */
3078	Assert(pCtx->cs.Attr.n.u1Present);
3079	Assert(!(pCtx->cs.Attr.u & 0xf00));
3080	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3081	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3082	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3083	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3084	\|\| (pCtx->cs.Attr.n.u1Granularity));
3085	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3086	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3087	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3088	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3089	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3090	else
3091	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3092	/* SS */
3093	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3094	{
3095	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3096	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3097	Assert(pCtx->ss.Attr.n.u1Present);
3098	Assert(!(pCtx->ss.Attr.u & 0xf00));
3099	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3100	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3101	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3102	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3103	\|\| (pCtx->ss.Attr.n.u1Granularity));
3104	}
3105	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3106	/* CR0 might not be up-to-date here always, hence disabled. */
3107	#if 0
3108	if (!pCtx->cr0 & X86_CR0_PE)
3109	Assert(!pCtx->ss.Attr.n.u2Dpl);
3110	#endif
3111	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3112	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3113	{
3114	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3115	Assert(pCtx->ds.Attr.n.u1Present);
3116	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3117	Assert(!(pCtx->ds.Attr.u & 0xf00));
3118	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3119	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3120	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3121	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3122	\|\| (pCtx->ds.Attr.n.u1Granularity));
3123	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3124	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3125	}
3126	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3127	{
3128	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3129	Assert(pCtx->es.Attr.n.u1Present);
3130	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3131	Assert(!(pCtx->es.Attr.u & 0xf00));
3132	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3133	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3134	\|\| !(pCtx->es.Attr.n.u1Granularity));
3135	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3136	\|\| (pCtx->es.Attr.n.u1Granularity));
3137	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3138	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3139	}
3140	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3141	{
3142	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3143	Assert(pCtx->fs.Attr.n.u1Present);
3144	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3145	Assert(!(pCtx->fs.Attr.u & 0xf00));
3146	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3147	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3148	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3149	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3150	\|\| (pCtx->fs.Attr.n.u1Granularity));
3151	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3152	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3153	}
3154	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3155	{
3156	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3157	Assert(pCtx->gs.Attr.n.u1Present);
3158	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3159	Assert(!(pCtx->gs.Attr.u & 0xf00));
3160	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3161	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3162	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3163	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3164	\|\| (pCtx->gs.Attr.n.u1Granularity));
3165	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3166	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3167	}
3168	/* 64-bit capable CPUs. */
3169	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3170	Assert(!(pCtx->cs.u64Base >> 32));
3171	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3172	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3173	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3174	# endif
3175	}
3176	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3177	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3178	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3179	{
3180	/* Real and v86 mode checks. */
3181	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3182	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3183	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3184	{
3185	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3186	}
3187	else
3188	{
3189	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3190	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3191	}
3192
3193	/* CS */
3194	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3195	Assert(pCtx->cs.u32Limit == 0xffff);
3196	Assert(u32CSAttr == 0xf3);
3197	/* SS */
3198	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3199	Assert(pCtx->ss.u32Limit == 0xffff);
3200	Assert(u32SSAttr == 0xf3);
3201	/* DS */
3202	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3203	Assert(pCtx->ds.u32Limit == 0xffff);
3204	Assert(u32DSAttr == 0xf3);
3205	/* ES */
3206	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3207	Assert(pCtx->es.u32Limit == 0xffff);
3208	Assert(u32ESAttr == 0xf3);
3209	/* FS */
3210	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3211	Assert(pCtx->fs.u32Limit == 0xffff);
3212	Assert(u32FSAttr == 0xf3);
3213	/* GS */
3214	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3215	Assert(pCtx->gs.u32Limit == 0xffff);
3216	Assert(u32GSAttr == 0xf3);
3217	/* 64-bit capable CPUs. */
3218	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3219	Assert(!(pCtx->cs.u64Base >> 32));
3220	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3221	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3222	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3223	# endif
3224	}
3225	}
3226	#endif /* DEBUG */
3227
3228
3229	/**
3230	* Writes a guest segment register into the guest-state area in the VMCS.
3231	*
3232	* @returns VBox status code.
3233	* @param pVM Pointer to the VM.
3234	* @param pVCpu Pointer to the VMCPU.
3235	* @param idxSel Index of the selector in the VMCS.
3236	* @param idxLimit Index of the segment limit in the VMCS.
3237	* @param idxBase Index of the segment base in the VMCS.
3238	* @param idxAccess Index of the access rights of the segment in the VMCS.
3239	* @param pSelReg Pointer to the segment selector.
3240	* @param pCtx Pointer to the guest-CPU context.
3241	*
3242	* @remarks No-long-jump zone!!!
3243	*/
3244	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3245	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3246	{
3247	int rc;
3248	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3249	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3250	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3251	AssertRCReturn(rc, rc);
3252
3253	uint32_t u32Access = pSelReg->Attr.u;
3254	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3255	{
3256	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3257	u32Access = 0xf3;
3258	Assert(pVM->hm.s.vmx.pRealModeTSS);
3259	Assert(PDMVmmDevHeapIsEnabled(pVM));
3260	}
3261	else
3262	{
3263	/*
3264	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3265	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3266	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3267	* loaded in protected-mode have their attribute as 0.
3268	*/
3269	if (!u32Access)
3270	u32Access = VMX_SEL_UNUSABLE;
3271	}
3272
3273	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3274	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3275	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3276
3277	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3278	AssertRCReturn(rc, rc);
3279	return rc;
3280	}
3281
3282
3283	/**
3284	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3285	* into the guest-state area in the VMCS.
3286	*
3287	* @returns VBox status code.
3288	* @param pVM Pointer to the VM.
3289	* @param pVCPU Pointer to the VMCPU.
3290	* @param pCtx Pointer to the guest-CPU context.
3291	*
3292	* @remarks No-long-jump zone!!!
3293	* @remarks Requires RFLAGS (for debug assertions).
3294	*/
3295	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3296	{
3297	int rc = VERR_INTERNAL_ERROR_5;
3298
3299	/*
3300	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3301	*/
3302	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3303	{
3304	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3305	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3306	{
3307	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3308	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3309	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3310	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3311	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3312	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3313	}
3314
3315	#ifdef VBOX_WITH_REM
3316	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3317	{
3318	Assert(pVM->hm.s.vmx.pRealModeTSS);
3319	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3320	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3321	{
3322	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3323	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3324	&& enmGuestMode >= PGMMODE_PROTECTED)
3325	{
3326	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3327	in real-mode (e.g. OpenBSD 4.0) */
3328	REMFlushTBs(pVM);
3329	Log2(("Switch to protected mode detected!\n"));
3330	}
3331	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3332	}
3333	}
3334	#endif
3335	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3336	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3337	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3338	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3339	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3340	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3341	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3342	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3343	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3344	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3345	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3346	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3347	AssertRCReturn(rc, rc);
3348
3349	#ifdef DEBUG
3350	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3351	#endif
3352	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3353	}
3354
3355	/*
3356	* Guest TR.
3357	*/
3358	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3359	{
3360	/*
3361	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3362	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3363	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3364	*/
3365	uint16_t u16Sel = 0;
3366	uint32_t u32Limit = 0;
3367	uint64_t u64Base = 0;
3368	uint32_t u32AccessRights = 0;
3369
3370	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3371	{
3372	u16Sel = pCtx->tr.Sel;
3373	u32Limit = pCtx->tr.u32Limit;
3374	u64Base = pCtx->tr.u64Base;
3375	u32AccessRights = pCtx->tr.Attr.u;
3376	}
3377	else
3378	{
3379	Assert(pVM->hm.s.vmx.pRealModeTSS);
3380	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3381
3382	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3383	RTGCPHYS GCPhys;
3384	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3385	AssertRCReturn(rc, rc);
3386
3387	X86DESCATTR DescAttr;
3388	DescAttr.u = 0;
3389	DescAttr.n.u1Present = 1;
3390	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3391
3392	u16Sel = 0;
3393	u32Limit = HM_VTX_TSS_SIZE;
3394	u64Base = GCPhys; /* in real-mode phys = virt. */
3395	u32AccessRights = DescAttr.u;
3396	}
3397
3398	/* Validate. */
3399	Assert(!(u16Sel & RT_BIT(2)));
3400	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3401	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3402	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3403	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3404	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3405	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3406	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3407	Assert( (u32Limit & 0xfff) == 0xfff
3408	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3409	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3410	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3411
3412	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3413	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3414	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3415	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3416	AssertRCReturn(rc, rc);
3417
3418	Log2(("VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3419	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3420	}
3421
3422	/*
3423	* Guest GDTR.
3424	*/
3425	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3426	{
3427	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3428	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3429	AssertRCReturn(rc, rc);
3430
3431	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3432	Log2(("VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3433	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3434	}
3435
3436	/*
3437	* Guest LDTR.
3438	*/
3439	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3440	{
3441	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3442	uint32_t u32Access = 0;
3443	if (!pCtx->ldtr.Attr.u)
3444	u32Access = VMX_SEL_UNUSABLE;
3445	else
3446	u32Access = pCtx->ldtr.Attr.u;
3447
3448	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3449	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3450	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3451	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3452	AssertRCReturn(rc, rc);
3453
3454	/* Validate. */
3455	if (!(u32Access & VMX_SEL_UNUSABLE))
3456	{
3457	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3458	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3459	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3460	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3461	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3462	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3463	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3464	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3465	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3466	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3467	}
3468
3469	Log2(("VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3470	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3471	}
3472
3473	/*
3474	* Guest IDTR.
3475	*/
3476	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3477	{
3478	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3479	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3480	AssertRCReturn(rc, rc);
3481
3482	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3483	Log2(("VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3484	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3485	}
3486
3487	/*
3488	* Guest FS & GS base MSRs.
3489	* We already initialized the FS & GS base as part of the guest segment registers, but the guest's FS/GS base
3490	* MSRs might have changed (e.g. due to WRMSR) and we need to update the bases if that happened. These MSRs
3491	* are only available in 64-bit mode.
3492	*/
3493	/** @todo Avoid duplication of this code in assembly (see MYPUSHSEGS) - it
3494	* should not be necessary to do it in assembly again. */
3495	if (CPUMIsGuestInLongModeEx(pCtx))
3496	{
3497	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FS_BASE_MSR)
3498	{
3499	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
3500	AssertRCReturn(rc, rc);
3501	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_FS_BASE_MSR;
3502	}
3503
3504	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GS_BASE_MSR)
3505	{
3506	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
3507	AssertRCReturn(rc, rc);
3508	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GS_BASE_MSR;
3509	}
3510	}
3511	else
3512	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_FS_BASE_MSR \| HM_CHANGED_GUEST_GS_BASE_MSR);
3513
3514	return VINF_SUCCESS;
3515	}
3516
3517
3518	/**
3519	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3520	* areas. These MSRs will automatically be loaded to the host CPU on every
3521	* successful VM entry and stored from the host CPU on every successful VM exit.
3522	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3523	*
3524	* @returns VBox status code.
3525	* @param pVM Pointer to the VM.
3526	* @param pVCpu Pointer to the VMCPU.
3527	* @param pCtx Pointer to the guest-CPU context.
3528	*
3529	* @remarks No-long-jump zone!!!
3530	*/
3531	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3532	{
3533	AssertPtr(pVCpu);
3534	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3535
3536	/*
3537	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3538	*/
3539	int rc = VINF_SUCCESS;
3540	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3541	{
3542	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3543	unsigned cGuestMsrs = 0;
3544
3545	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3546	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3547	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3548	if (fSupportsNX \|\| fSupportsLongMode)
3549	{
3550	/** @todo support save IA32_EFER, i.e.
3551	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3552	* guest EFER need not be part of the VM-entry MSR-load area. */
3553	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3554	pGuestMsr->u32Reserved = 0;
3555	pGuestMsr->u64Value = pCtx->msrEFER;
3556	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3557	if (!CPUMIsGuestInLongModeEx(pCtx))
3558	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3559	pGuestMsr++; cGuestMsrs++;
3560	if (fSupportsLongMode)
3561	{
3562	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3563	pGuestMsr->u32Reserved = 0;
3564	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3565	pGuestMsr++; cGuestMsrs++;
3566	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3567	pGuestMsr->u32Reserved = 0;
3568	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3569	pGuestMsr++; cGuestMsrs++;
3570	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3571	pGuestMsr->u32Reserved = 0;
3572	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3573	pGuestMsr++; cGuestMsrs++;
3574	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3575	#if 0
3576	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3577	pGuestMsr->u32Reserved = 0;
3578	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3579	pGuestMsr++; cGuestMsrs++;
3580	#endif
3581	}
3582	}
3583
3584	/*
3585	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3586	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3587	*/
3588	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3589	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3590	{
3591	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3592	pGuestMsr->u32Reserved = 0;
3593	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3594	AssertRCReturn(rc, rc);
3595	pGuestMsr++; cGuestMsrs++;
3596	}
3597
3598	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3599	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3600	{
3601	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3602	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3603	}
3604
3605	/* Update the VCPU's copy of the guest MSR count. */
3606	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3607	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3608	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3609	AssertRCReturn(rc, rc);
3610
3611	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3612	}
3613
3614	/*
3615	* Guest Sysenter MSRs.
3616	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3617	* VM exits on WRMSRs for these MSRs.
3618	*/
3619	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3620	{
3621	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3622	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3623	}
3624	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3625	{
3626	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3627	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3628	}
3629	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3630	{
3631	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3632	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3633	}
3634	AssertRCReturn(rc, rc);
3635
3636	return rc;
3637	}
3638
3639
3640	/**
3641	* Loads the guest activity state into the guest-state area in the VMCS.
3642	*
3643	* @returns VBox status code.
3644	* @param pVM Pointer to the VM.
3645	* @param pVCpu Pointer to the VMCPU.
3646	* @param pCtx Pointer to the guest-CPU context.
3647	*
3648	* @remarks No-long-jump zone!!!
3649	*/
3650	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3651	{
3652	/** @todo See if we can make use of other states, e.g.
3653	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3654	int rc = VINF_SUCCESS;
3655	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3656	{
3657	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3658	AssertRCReturn(rc, rc);
3659	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3660	}
3661	return rc;
3662	}
3663
3664
3665	/**
3666	* Sets up the appropriate function to run guest code.
3667	*
3668	* @returns VBox status code.
3669	* @param pVM Pointer to the VM.
3670	* @param pVCpu Pointer to the VMCPU.
3671	* @param pCtx Pointer to the guest-CPU context.
3672	*
3673	* @remarks No-long-jump zone!!!
3674	*/
3675	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3676	{
3677	if (CPUMIsGuestInLongModeEx(pCtx))
3678	{
3679	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3680	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3681	#endif
3682	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3683	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3684	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3685	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3686	#else
3687	/* 64-bit host or hybrid host. */
3688	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3689	#endif
3690	}
3691	else
3692	{
3693	/* Guest is not in long mode, use the 32-bit handler. */
3694	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3695	}
3696	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3697	return VINF_SUCCESS;
3698	}
3699
3700
3701	/**
3702	* Wrapper for running the guest code in VT-x.
3703	*
3704	* @returns VBox strict status code.
3705	* @param pVM Pointer to the VM.
3706	* @param pVCpu Pointer to the VMCPU.
3707	* @param pCtx Pointer to the guest-CPU context.
3708	*
3709	* @remarks No-long-jump zone!!!
3710	*/
3711	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3712	{
3713	/*
3714	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3715	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3716	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3717	*/
3718	#ifdef VBOX_WITH_KERNEL_USING_XMM
3719	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3720	#else
3721	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3722	#endif
3723	}
3724
3725
3726	/**
3727	* Report world-switch error and dump some useful debug info.
3728	*
3729	* @param pVM Pointer to the VM.
3730	* @param pVCpu Pointer to the VMCPU.
3731	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3732	* @param pCtx Pointer to the guest-CPU context.
3733	* @param pVmxTransient Pointer to the VMX transient structure (only
3734	* exitReason updated).
3735	*/
3736	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3737	{
3738	Assert(pVM);
3739	Assert(pVCpu);
3740	Assert(pCtx);
3741	Assert(pVmxTransient);
3742	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3743
3744	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3745	switch (rcVMRun)
3746	{
3747	case VERR_VMX_INVALID_VMXON_PTR:
3748	AssertFailed();
3749	break;
3750	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3751	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3752	{
3753	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3754	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3755	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3756	AssertRC(rc);
3757
3758	#ifdef VBOX_STRICT
3759	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3760	pVmxTransient->uExitReason));
3761	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3762	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3763	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3764	Log(("InstrError Desc. \"%s\"\n", s_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3765	else
3766	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3767
3768	/* VMX control bits. */
3769	uint32_t u32Val;
3770	uint64_t u64Val;
3771	RTHCUINTREG uHCReg;
3772	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3773	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3774	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3775	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3776	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3777	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3778	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3779	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3780	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3781	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3782	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3783	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3784	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3785	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3786	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3787	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3788	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3789	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3790	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3791	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3792	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3793	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3794	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3795	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3796	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3797	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3798	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3799	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3800	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3801	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3802	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3803	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3804	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3805	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3806	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3807	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3808	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3809	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3810	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3811	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3812	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3813	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3814
3815	/* Guest bits. */
3816	RTGCUINTREG uGCReg;
3817	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3818	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3819	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3820	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3821	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3822	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3823	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3824	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3825
3826	/* Host bits. */
3827	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3828	Log(("Host CR0 %#RHr\n", uHCReg));
3829	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3830	Log(("Host CR3 %#RHr\n", uHCReg));
3831	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3832	Log(("Host CR4 %#RHr\n", uHCReg));
3833
3834	RTGDTR HostGdtr;
3835	PCX86DESCHC pDesc;
3836	ASMGetGDTR(&HostGdtr);
3837	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3838	Log(("Host CS %#08x\n", u32Val));
3839	if (u32Val < HostGdtr.cbGdt)
3840	{
3841	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3842	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3843	}
3844
3845	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3846	Log(("Host DS %#08x\n", u32Val));
3847	if (u32Val < HostGdtr.cbGdt)
3848	{
3849	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3850	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3851	}
3852
3853	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3854	Log(("Host ES %#08x\n", u32Val));
3855	if (u32Val < HostGdtr.cbGdt)
3856	{
3857	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3858	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3859	}
3860
3861	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3862	Log(("Host FS %#08x\n", u32Val));
3863	if (u32Val < HostGdtr.cbGdt)
3864	{
3865	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3866	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3867	}
3868
3869	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3870	Log(("Host GS %#08x\n", u32Val));
3871	if (u32Val < HostGdtr.cbGdt)
3872	{
3873	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3874	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3875	}
3876
3877	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3878	Log(("Host SS %#08x\n", u32Val));
3879	if (u32Val < HostGdtr.cbGdt)
3880	{
3881	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3882	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3883	}
3884
3885	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3886	Log(("Host TR %#08x\n", u32Val));
3887	if (u32Val < HostGdtr.cbGdt)
3888	{
3889	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3890	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3891	}
3892
3893	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3894	Log(("Host TR Base %#RHv\n", uHCReg));
3895	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3896	Log(("Host GDTR Base %#RHv\n", uHCReg));
3897	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3898	Log(("Host IDTR Base %#RHv\n", uHCReg));
3899	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3900	Log(("Host SYSENTER CS %#08x\n", u32Val));
3901	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3902	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3903	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3904	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3905	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3906	Log(("Host RSP %#RHv\n", uHCReg));
3907	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3908	Log(("Host RIP %#RHv\n", uHCReg));
3909	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3910	if (VMX_IS_64BIT_HOST_MODE())
3911	{
3912	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3913	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3914	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3915	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3916	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3917	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3918	}
3919	# endif
3920	#endif /* VBOX_STRICT */
3921	break;
3922	}
3923
3924	default:
3925	/* Impossible */
3926	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3927	break;
3928	}
3929	NOREF(pVM);
3930	}
3931
3932
3933	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3934	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3935	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3936	#endif
3937
3938	#ifdef VBOX_STRICT
3939	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3940	{
3941	switch (idxField)
3942	{
3943	case VMX_VMCS_GUEST_RIP:
3944	case VMX_VMCS_GUEST_RSP:
3945	case VMX_VMCS_GUEST_RFLAGS:
3946	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
3947	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
3948	case VMX_VMCS_GUEST_CR0:
3949	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
3950	case VMX_VMCS_GUEST_CR4:
3951	case VMX_VMCS_GUEST_DR7:
3952	case VMX_VMCS32_GUEST_SYSENTER_CS:
3953	case VMX_VMCS_GUEST_SYSENTER_EIP:
3954	case VMX_VMCS_GUEST_SYSENTER_ESP:
3955	case VMX_VMCS32_GUEST_GDTR_LIMIT:
3956	case VMX_VMCS_GUEST_GDTR_BASE:
3957	case VMX_VMCS32_GUEST_IDTR_LIMIT:
3958	case VMX_VMCS_GUEST_IDTR_BASE:
3959	case VMX_VMCS16_GUEST_FIELD_CS:
3960	case VMX_VMCS32_GUEST_CS_LIMIT:
3961	case VMX_VMCS_GUEST_CS_BASE:
3962	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
3963	case VMX_VMCS16_GUEST_FIELD_DS:
3964	case VMX_VMCS32_GUEST_DS_LIMIT:
3965	case VMX_VMCS_GUEST_DS_BASE:
3966	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
3967	case VMX_VMCS16_GUEST_FIELD_ES:
3968	case VMX_VMCS32_GUEST_ES_LIMIT:
3969	case VMX_VMCS_GUEST_ES_BASE:
3970	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
3971	case VMX_VMCS16_GUEST_FIELD_FS:
3972	case VMX_VMCS32_GUEST_FS_LIMIT:
3973	case VMX_VMCS_GUEST_FS_BASE:
3974	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
3975	case VMX_VMCS16_GUEST_FIELD_GS:
3976	case VMX_VMCS32_GUEST_GS_LIMIT:
3977	case VMX_VMCS_GUEST_GS_BASE:
3978	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
3979	case VMX_VMCS16_GUEST_FIELD_SS:
3980	case VMX_VMCS32_GUEST_SS_LIMIT:
3981	case VMX_VMCS_GUEST_SS_BASE:
3982	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
3983	case VMX_VMCS16_GUEST_FIELD_LDTR:
3984	case VMX_VMCS32_GUEST_LDTR_LIMIT:
3985	case VMX_VMCS_GUEST_LDTR_BASE:
3986	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
3987	case VMX_VMCS16_GUEST_FIELD_TR:
3988	case VMX_VMCS32_GUEST_TR_LIMIT:
3989	case VMX_VMCS_GUEST_TR_BASE:
3990	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
3991	case VMX_VMCS32_RO_EXIT_REASON:
3992	case VMX_VMCS32_RO_VM_INSTR_ERROR:
3993	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
3994	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
3995	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
3996	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
3997	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3998	case VMX_VMCS32_RO_IDT_INFO:
3999	case VMX_VMCS32_RO_IDT_ERROR_CODE:
4000	case VMX_VMCS_GUEST_CR3:
4001	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4002	return true;
4003	}
4004	return false;
4005	}
4006
4007	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4008	{
4009	switch (idxField)
4010	{
4011	case VMX_VMCS_GUEST_LDTR_BASE:
4012	case VMX_VMCS_GUEST_TR_BASE:
4013	case VMX_VMCS_GUEST_GDTR_BASE:
4014	case VMX_VMCS_GUEST_IDTR_BASE:
4015	case VMX_VMCS_GUEST_SYSENTER_EIP:
4016	case VMX_VMCS_GUEST_SYSENTER_ESP:
4017	case VMX_VMCS_GUEST_CR0:
4018	case VMX_VMCS_GUEST_CR4:
4019	case VMX_VMCS_GUEST_CR3:
4020	case VMX_VMCS_GUEST_DR7:
4021	case VMX_VMCS_GUEST_RIP:
4022	case VMX_VMCS_GUEST_RSP:
4023	case VMX_VMCS_GUEST_CS_BASE:
4024	case VMX_VMCS_GUEST_DS_BASE:
4025	case VMX_VMCS_GUEST_ES_BASE:
4026	case VMX_VMCS_GUEST_FS_BASE:
4027	case VMX_VMCS_GUEST_GS_BASE:
4028	case VMX_VMCS_GUEST_SS_BASE:
4029	return true;
4030	}
4031	return false;
4032	}
4033	#endif /* VBOX_STRICT */
4034
4035	/**
4036	* Executes the specified handler in 64-bit mode.
4037	*
4038	* @returns VBox status code.
4039	* @param pVM Pointer to the VM.
4040	* @param pVCpu Pointer to the VMCPU.
4041	* @param pCtx Pointer to the guest CPU context.
4042	* @param pfnHandler Pointer to the RC handler function.
4043	* @param cbParam Number of parameters.
4044	* @param paParam Array of 32-bit parameters.
4045	*/
4046	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
4047	uint32_t *paParam)
4048	{
4049	int rc, rc2;
4050	PHMGLOBLCPUINFO pCpu;
4051	RTHCPHYS HCPhysCpuPage;
4052	RTHCUINTREG uOldEFlags;
4053
4054	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4055	Assert(pfnHandler);
4056	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4057	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4058
4059	#ifdef VBOX_STRICT
4060	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4061	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4062
4063	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4064	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4065	#endif
4066
4067	/* Disable interrupts. */
4068	uOldEFlags = ASMIntDisableFlags();
4069
4070	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4071	RTCPUID idHostCpu = RTMpCpuId();
4072	CPUMR0SetLApic(pVM, idHostCpu);
4073	#endif
4074
4075	pCpu = HMR0GetCurrentCpu();
4076	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4077
4078	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4079	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4080
4081	/* Leave VMX Root Mode. */
4082	VMXDisable();
4083
4084	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4085
4086	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4087	CPUMSetHyperEIP(pVCpu, pfnHandler);
4088	for (int i = (int)cbParam - 1; i >= 0; i--)
4089	CPUMPushHyper(pVCpu, paParam[i]);
4090
4091	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4092
4093	/* Call the switcher. */
4094	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4095	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4096
4097	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4098	/* Make sure the VMX instructions don't cause #UD faults. */
4099	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4100
4101	/* Re-enter VMX Root Mode */
4102	rc2 = VMXEnable(HCPhysCpuPage);
4103	if (RT_FAILURE(rc2))
4104	{
4105	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4106	ASMSetFlags(uOldEFlags);
4107	return rc2;
4108	}
4109
4110	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4111	AssertRC(rc2);
4112	Assert(!(ASMGetFlags() & X86_EFL_IF));
4113	ASMSetFlags(uOldEFlags);
4114	return rc;
4115	}
4116
4117
4118	/**
4119	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4120	* supporting 64-bit guests.
4121	*
4122	* @returns VBox status code.
4123	* @param fResume Whether to VMLAUNCH or VMRESUME.
4124	* @param pCtx Pointer to the guest-CPU context.
4125	* @param pCache Pointer to the VMCS cache.
4126	* @param pVM Pointer to the VM.
4127	* @param pVCpu Pointer to the VMCPU.
4128	*/
4129	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4130	{
4131	uint32_t aParam[6];
4132	PHMGLOBLCPUINFO pCpu = NULL;
4133	RTHCPHYS HCPhysCpuPage = 0;
4134	int rc = VERR_INTERNAL_ERROR_5;
4135
4136	pCpu = HMR0GetCurrentCpu();
4137	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4138
4139	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4140	pCache->uPos = 1;
4141	pCache->interPD = PGMGetInterPaeCR3(pVM);
4142	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4143	#endif
4144
4145	#ifdef DEBUG
4146	pCache->TestIn.HCPhysCpuPage= 0;
4147	pCache->TestIn.HCPhysVmcs = 0;
4148	pCache->TestIn.pCache = 0;
4149	pCache->TestOut.HCPhysVmcs = 0;
4150	pCache->TestOut.pCache = 0;
4151	pCache->TestOut.pCtx = 0;
4152	pCache->TestOut.eflags = 0;
4153	#endif
4154
4155	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4156	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4157	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4158	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4159	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4160	aParam[5] = 0;
4161
4162	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4163	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4164	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4165	#endif
4166	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4167
4168	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4169	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4170	Assert(pCtx->dr[4] == 10);
4171	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4172	#endif
4173
4174	#ifdef DEBUG
4175	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4176	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4177	pVCpu->hm.s.vmx.HCPhysVmcs));
4178	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4179	pCache->TestOut.HCPhysVmcs));
4180	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4181	pCache->TestOut.pCache));
4182	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4183	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4184	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4185	pCache->TestOut.pCtx));
4186	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4187	#endif
4188	return rc;
4189	}
4190
4191
4192	/**
4193	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4194	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4195	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4196	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4197	*
4198	* @returns VBox status code.
4199	* @param pVM Pointer to the VM.
4200	* @param pVCpu Pointer to the VMCPU.
4201	*/
4202	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4203	{
4204	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4205	{ \
4206	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4207	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4208	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4209	}
4210
4211	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4212	{ \
4213	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4214	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4215	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4216	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4217	}
4218
4219	AssertPtr(pVM);
4220	AssertPtr(pVCpu);
4221	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4222
4223	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4224	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4225	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4226	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4227	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4228	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4229	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4230	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4231	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4232
4233	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4234	#if 0
4235	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4236	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4237	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4238	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4239	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4240	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4241	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4242	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4243	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4244	#endif
4245
4246	/* 32-bit guest-state fields. */
4247	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4248	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4249	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4250	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4251	/* Unused 32-bit guest-state fields. */
4252	#if 0
4253	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4254	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4255	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE);
4256	#endif
4257
4258	/* Natural width guest-state fields. */
4259	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4260	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4261	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4262	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4263	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4264	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4265	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4267	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4268	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4275	/* Unused natural width guest-state fields. */
4276	#if 0
4277	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4279	#endif
4280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4282
4283	if (pVM->hm.s.fNestedPaging)
4284	{
4285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4287	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4288	}
4289	else
4290	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4291
4292	#undef VMXLOCAL_INIT_VMCS_SELREG
4293	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4294	return VINF_SUCCESS;
4295	}
4296
4297
4298	/**
4299	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4300	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4301	* darwin, running 64-bit guests).
4302	*
4303	* @returns VBox status code.
4304	* @param pVCpu Pointer to the VMCPU.
4305	* @param idxField The VMCS field encoding.
4306	* @param u64Val 16, 32 or 64 bits value.
4307	*/
4308	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4309	{
4310	int rc;
4311	switch (idxField)
4312	{
4313	/*
4314	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4315	*/
4316	/* 64-bit Control fields. */
4317	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4318	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4319	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4320	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4321	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4322	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4323	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4324	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4325	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4326	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4327	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4328	case VMX_VMCS64_CTRL_EPTP_FULL:
4329	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4330	/* 64-bit Read-only data fields. */
4331	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4332	/* 64-bit Guest-state fields. */
4333	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4334	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4335	case VMX_VMCS64_GUEST_PAT_FULL:
4336	case VMX_VMCS64_GUEST_EFER_FULL:
4337	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4338	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4339	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4340	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4341	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4342	/* 64-bit Host-state fields. */
4343	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4344	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4345	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4346	{
4347	rc = VMXWriteVmcs32(idxField, u64Val);
4348	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4349	break;
4350	}
4351
4352	/*
4353	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4354	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4355	*/
4356	/* Natural-width Guest-state fields. */
4357	case VMX_VMCS_GUEST_CR0:
4358	case VMX_VMCS_GUEST_CR3:
4359	case VMX_VMCS_GUEST_CR4:
4360	case VMX_VMCS_GUEST_ES_BASE:
4361	case VMX_VMCS_GUEST_CS_BASE:
4362	case VMX_VMCS_GUEST_SS_BASE:
4363	case VMX_VMCS_GUEST_DS_BASE:
4364	case VMX_VMCS_GUEST_FS_BASE:
4365	case VMX_VMCS_GUEST_GS_BASE:
4366	case VMX_VMCS_GUEST_LDTR_BASE:
4367	case VMX_VMCS_GUEST_TR_BASE:
4368	case VMX_VMCS_GUEST_GDTR_BASE:
4369	case VMX_VMCS_GUEST_IDTR_BASE:
4370	case VMX_VMCS_GUEST_DR7:
4371	case VMX_VMCS_GUEST_RSP:
4372	case VMX_VMCS_GUEST_RIP:
4373	case VMX_VMCS_GUEST_RFLAGS:
4374	case VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS:
4375	case VMX_VMCS_GUEST_SYSENTER_ESP:
4376	case VMX_VMCS_GUEST_SYSENTER_EIP:
4377	{
4378	if ((u64Val >> 32ULL) == 0)
4379	{
4380	/* If this field is 64-bit, VT-x will zero out the top bits. */
4381	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4382	}
4383	else
4384	{
4385	/* Assert that only the 32->64 switcher case should ever come here. */
4386	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4387	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4388	}
4389	break;
4390	}
4391
4392	default:
4393	{
4394	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4395	rc = VERR_INVALID_PARAMETER;
4396	break;
4397	}
4398	}
4399	AssertRCReturn(rc, rc);
4400	return rc;
4401	}
4402
4403
4404	/**
4405	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4406	* hosts (except darwin) for 64-bit guests.
4407	*
4408	* @param pVCpu Pointer to the VMCPU.
4409	* @param idxField The VMCS field encoding.
4410	* @param u64Val 16, 32 or 64 bits value.
4411	*/
4412	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4413	{
4414	AssertPtr(pVCpu);
4415	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4416
4417	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4418	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4419
4420	/* Make sure there are no duplicates. */
4421	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
4422	{
4423	if (pCache->Write.aField[i] == idxField)
4424	{
4425	pCache->Write.aFieldVal[i] = u64Val;
4426	return VINF_SUCCESS;
4427	}
4428	}
4429
4430	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4431	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4432	pCache->Write.cValidEntries++;
4433	return VINF_SUCCESS;
4434	}
4435
4436
4437	/**
4438	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4439	*
4440	* @param pVCpu Pointer to the VMCPU.
4441	* @param pCache Pointer to the VMCS cache.
4442	*
4443	* @remarks No-long-jump zone!!!
4444	*/
4445	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4446	{
4447	AssertPtr(pCache);
4448	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4449	{
4450	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4451	AssertRC(rc);
4452	}
4453	pCache->Write.cValidEntries = 0;
4454	}
4455
4456
4457	/**
4458	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4459	*
4460	* @param pVCpu Pointer to the VMCPU.
4461	* @param pCache Pointer to the VMCS cache.
4462	*
4463	* @remarks No-long-jump zone!!!
4464	*/
4465	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4466	{
4467	AssertPtr(pCache);
4468	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4469	{
4470	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4471	AssertRC(rc);
4472	}
4473	}
4474	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4475
4476
4477	/**
4478	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4479	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4480	* timer.
4481	*
4482	* @returns VBox status code.
4483	* @param pVM Pointer to the VM.
4484	* @param pVCpu Pointer to the VMCPU.
4485	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4486	* out-of-sync. Make sure to update the required fields
4487	* before using them.
4488	* @remarks No-long-jump zone!!!
4489	*/
4490	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4491	{
4492	int rc = VERR_INTERNAL_ERROR_5;
4493	bool fOffsettedTsc = false;
4494	if (pVM->hm.s.vmx.fUsePreemptTimer)
4495	{
4496	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4497
4498	/* Make sure the returned values have sane upper and lower boundaries. */
4499	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4500	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4501	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4502	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4503
4504	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4505	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4506	}
4507	else
4508	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4509
4510	if (fOffsettedTsc)
4511	{
4512	uint64_t u64CurTSC = ASMReadTSC();
4513	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4514	{
4515	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4516	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4517
4518	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4519	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4520	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4521	}
4522	else
4523	{
4524	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4525	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4526	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4527	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4528	}
4529	}
4530	else
4531	{
4532	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4533	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4534	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4535	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4536	}
4537	}
4538
4539
4540	/**
4541	* Determines if an exception is a benign exception. Benign exceptions
4542	* are ones which cannot cause double-faults.
4543	*
4544	* @returns true if the exception is benign, false otherwise.
4545	* @param uVector The exception vector.
4546	*/
4547	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint32_t uVector)
4548	{
4549	switch (uVector)
4550	{
4551	case X86_XCPT_DB:
4552	case X86_XCPT_NMI:
4553	case X86_XCPT_BP:
4554	case X86_XCPT_OF:
4555	case X86_XCPT_BR:
4556	case X86_XCPT_UD:
4557	case X86_XCPT_NM:
4558	case X86_XCPT_CO_SEG_OVERRUN:
4559	case X86_XCPT_MF:
4560	case X86_XCPT_AC:
4561	case X86_XCPT_MC:
4562	case X86_XCPT_XF:
4563	return true;
4564	default:
4565	return false;
4566	}
4567	}
4568
4569
4570	/**
4571	* Determines if an exception is a contributory exception. Contributory
4572	* exceptions are ones which can cause double-faults.
4573	*
4574	* @returns true if the exception is contributory, false otherwise.
4575	* @param uVector The exception vector.
4576	*/
4577	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4578	{
4579	switch (uVector)
4580	{
4581	case X86_XCPT_GP:
4582	case X86_XCPT_SS:
4583	case X86_XCPT_NP:
4584	case X86_XCPT_TS:
4585	case X86_XCPT_DE:
4586	return true;
4587	default:
4588	return false;
4589	}
4590	return false;
4591	}
4592
4593
4594	/**
4595	* Determines if we are intercepting any contributory exceptions.
4596	*
4597	* @returns true if we are intercepting any contributory exception, false
4598	* otherwise.
4599	* @param pVCpu Pointer to the VMCPU.
4600	*/
4601	DECLINLINE(bool) hmR0VmxInterceptingContributoryXcpts(PVMCPU pVCpu)
4602	{
4603	if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_GP))
4604	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_SS))
4605	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_NP))
4606	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_TS))
4607	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_DE)))
4608	{
4609	return true;
4610	}
4611	return false;
4612	}
4613
4614
4615	/**
4616	* Handle a condition that occurred while delivering an event through the guest
4617	* IDT.
4618	*
4619	* @returns VBox status code (informational error codes included).
4620	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4621	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4622	* continue execution of the guest which will delivery the #DF.
4623	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4624	*
4625	* @param pVM Pointer to the VM.
4626	* @param pVCpu Pointer to the VMCPU.
4627	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4628	* out-of-sync. Make sure to update the required fields
4629	* before using them.
4630	* @param pVmxTransient Pointer to the VMX transient structure.
4631	*
4632	* @remarks No-long-jump zone!!!
4633	*/
4634	static int hmR0VmxCheckExitDueToEventDelivery(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4635	{
4636	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4637	AssertRCReturn(rc, rc);
4638	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4639	{
4640	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4641	AssertRCReturn(rc, rc);
4642
4643	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4644	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4645	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4646
4647	typedef enum
4648	{
4649	VMXREFLECTXCPT_XCPT, /* Reflect Idt-vectoring exception. */
4650	VMXREFLECTXCPT_DF, /* Reflect a double-fault to the guest. */
4651	VMXREFLECTXCPT_TF, /* Reflect a triple fault state to the VMM. */
4652	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4653	} VMXREFLECTXCPT;
4654
4655	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4656	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4657	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4658	{
4659	if ( hmR0VmxIsBenignXcpt(uIdtVector)
4660	\|\| hmR0VmxIsBenignXcpt(uExitVector)
4661	\|\| ( hmR0VmxIsContributoryXcpt(uIdtVector)
4662	&& uExitVector == X86_XCPT_PF))
4663	{
4664	enmReflect = VMXREFLECTXCPT_XCPT;
4665	}
4666	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF))
4667	&& uIdtVector == X86_XCPT_PF
4668	&& uExitVector == X86_XCPT_PF)
4669	{
4670	pVmxTransient->fVectoringPF = true;
4671	}
4672	else if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4673	&& hmR0VmxIsContributoryXcpt(uExitVector))
4674	{
4675	enmReflect = VMXREFLECTXCPT_DF;
4676	}
4677	else if ( hmR0VmxInterceptingContributoryXcpts(pVCpu)
4678	&& uIdtVector == X86_XCPT_PF
4679	&& hmR0VmxIsContributoryXcpt(uExitVector))
4680	{
4681	enmReflect = VMXREFLECTXCPT_DF;
4682	}
4683	else if (uIdtVector == X86_XCPT_DF)
4684	enmReflect = VMXREFLECTXCPT_TF;
4685	}
4686	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4687	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4688	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4689	{
4690	/*
4691	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4692	* (whatever they are) as they reoccur when restarting the instruction.
4693	*/
4694	enmReflect = VMXREFLECTXCPT_XCPT;
4695	}
4696
4697	Assert(pVmxTransient->fVectoringPF == false \|\| enmReflect == VMXREFLECTXCPT_NONE);
4698	switch (enmReflect)
4699	{
4700	case VMXREFLECTXCPT_XCPT:
4701	{
4702	Assert(!pVCpu->hm.s.Event.fPending);
4703	pVCpu->hm.s.Event.fPending = true;
4704	pVCpu->hm.s.Event.u64IntrInfo = VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo);
4705	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4706	{
4707	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4708	AssertRCReturn(rc, rc);
4709	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4710	}
4711	else
4712	pVCpu->hm.s.Event.u32ErrCode = 0;
4713	Log(("Pending event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4714	break;
4715	}
4716
4717	case VMXREFLECTXCPT_DF:
4718	{
4719	uint32_t u32IntrInfo;
4720	u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4721	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4722	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4723
4724	Assert(!pVCpu->hm.s.Event.fPending);
4725	pVCpu->hm.s.Event.fPending = true;
4726	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4727	pVCpu->hm.s.Event.u32ErrCode = 0;
4728	rc = VINF_VMX_DOUBLE_FAULT;
4729	Log(("Pending #DF %#RX64 uIdt=%#x uExit=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4730	break;
4731	}
4732
4733	case VMXREFLECTXCPT_TF:
4734	{
4735	Log(("Pending triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4736	rc = VINF_EM_RESET;
4737	break;
4738	}
4739
4740	default: /* shut up gcc. */
4741	break;
4742	}
4743	}
4744	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET \|\| rc == VINF_VMX_DOUBLE_FAULT);
4745	return rc;
4746	}
4747
4748
4749	/**
4750	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4751	*
4752	* @returns VBox status code.
4753	* @param pVM Pointer to the VM.
4754	* @param pVCpu Pointer to the VMCPU.
4755	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4756	* out-of-sync. Make sure to update the required fields
4757	* before using them.
4758	*
4759	* @remarks No-long-jump zone!!!
4760	*/
4761	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4762	{
4763	int rc = VINF_SUCCESS;
4764	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4765	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4766	{
4767	RTGCUINTREG uVal = 0;
4768	RTCCUINTREG uShadow = 0;
4769	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4770	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4771	AssertRCReturn(rc, rc);
4772	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4773	CPUMSetGuestCR0(pVCpu, uVal);
4774	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4775	}
4776	return rc;
4777	}
4778
4779
4780	/**
4781	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4782	*
4783	* @returns VBox status code.
4784	* @param pVM Pointer to the VM.
4785	* @param pVCpu Pointer to the VMCPU.
4786	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4787	* out-of-sync. Make sure to update the required fields
4788	* before using them.
4789	*
4790	* @remarks No-long-jump zone!!!
4791	*/
4792	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4793	{
4794	int rc = VINF_SUCCESS;
4795	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4796	{
4797	RTGCUINTREG uVal = 0;
4798	RTCCUINTREG uShadow = 0;
4799	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4800	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4801	AssertRCReturn(rc, rc);
4802	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4803	CPUMSetGuestCR4(pVCpu, uVal);
4804	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4805	}
4806	return rc;
4807	}
4808
4809
4810	/**
4811	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4812	*
4813	* @returns VBox status code.
4814	* @param pVM Pointer to the VM.
4815	* @param pVCpu Pointer to the VMCPU.
4816	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4817	* out-of-sync. Make sure to update the required fields
4818	* before using them.
4819	*
4820	* @remarks No-long-jump zone!!!
4821	*/
4822	DECLINLINE(int) hmR0VmxSaveGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4823	{
4824	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4825	return VINF_SUCCESS;
4826
4827	RTGCUINTREG uVal = 0;
4828	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4829	AssertRCReturn(rc, rc);
4830	pMixedCtx->rip = uVal;
4831	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4832	return rc;
4833	}
4834
4835
4836	/**
4837	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4838	*
4839	* @returns VBox status code.
4840	* @param pVM Pointer to the VM.
4841	* @param pVCpu Pointer to the VMCPU.
4842	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4843	* out-of-sync. Make sure to update the required fields
4844	* before using them.
4845	*
4846	* @remarks No-long-jump zone!!!
4847	*/
4848	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4849	{
4850	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4851	return VINF_SUCCESS;
4852
4853	RTGCUINTREG uVal = 0;
4854	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4855	AssertRCReturn(rc, rc);
4856	pMixedCtx->rsp = uVal;
4857	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4858	return rc;
4859	}
4860
4861
4862	/**
4863	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4864	*
4865	* @returns VBox status code.
4866	* @param pVM Pointer to the VM.
4867	* @param pVCpu Pointer to the VMCPU.
4868	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4869	* out-of-sync. Make sure to update the required fields
4870	* before using them.
4871	*
4872	* @remarks No-long-jump zone!!!
4873	*/
4874	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4875	{
4876	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4877	return VINF_SUCCESS;
4878
4879	RTGCUINTREG uVal = 0;
4880	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4881	AssertRCReturn(rc, rc);
4882	pMixedCtx->rflags.u64 = uVal;
4883
4884	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4885	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4886	{
4887	Assert(pVM->hm.s.vmx.pRealModeTSS);
4888	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4889	pMixedCtx->eflags.Bits.u1VM = 0;
4890	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4891	}
4892
4893	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4894	return rc;
4895	}
4896
4897
4898	/**
4899	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4900	* guest-CPU context.
4901	*/
4902	static int hmR0VmxSaveGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4903	{
4904	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4905	rc \|= hmR0VmxSaveGuestRsp(pVM, pVCpu, pMixedCtx);
4906	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
4907	return rc;
4908	}
4909
4910
4911	/**
4912	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4913	* from the guest-state area in the VMCS.
4914	*
4915	* @param pVM Pointer to the VM.
4916	* @param pVCpu Pointer to the VMCPU.
4917	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4918	* out-of-sync. Make sure to update the required fields
4919	* before using them.
4920	*
4921	* @remarks No-long-jump zone!!!
4922	*/
4923	DECLINLINE(void) hmR0VmxSaveGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4924	{
4925	uint32_t uIntrState = 0;
4926	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4927	AssertRC(rc);
4928
4929	if (!uIntrState)
4930	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4931	else
4932	{
4933	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4934	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4935	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4936	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* for hmR0VmxLoadGuestIntrState(). */
4937	AssertRC(rc);
4938	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4939	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4940	}
4941	}
4942
4943
4944	/**
4945	* Saves the guest's activity state.
4946	*
4947	* @returns VBox status code.
4948	* @param pVM Pointer to the VM.
4949	* @param pVCpu Pointer to the VMCPU.
4950	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4951	* out-of-sync. Make sure to update the required fields
4952	* before using them.
4953	*
4954	* @remarks No-long-jump zone!!!
4955	*/
4956	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4957	{
4958	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4959	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
4960	return VINF_SUCCESS;
4961	}
4962
4963
4964	/**
4965	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4966	* the current VMCS into the guest-CPU context.
4967	*
4968	* @returns VBox status code.
4969	* @param pVM Pointer to the VM.
4970	* @param pVCpu Pointer to the VMCPU.
4971	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4972	* out-of-sync. Make sure to update the required fields
4973	* before using them.
4974	*
4975	* @remarks No-long-jump zone!!!
4976	*/
4977	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4978	{
4979	int rc = VINF_SUCCESS;
4980	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4981	{
4982	uint32_t u32Val = 0;
4983	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4984	pMixedCtx->SysEnter.cs = u32Val;
4985	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4986	}
4987
4988	RTGCUINTREG uGCVal = 0;
4989	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4990	{
4991	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4992	pMixedCtx->SysEnter.eip = uGCVal;
4993	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4994	}
4995	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4996	{
4997	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4998	pMixedCtx->SysEnter.esp = uGCVal;
4999	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5000	}
5001	return rc;
5002	}
5003
5004
5005	/**
5006	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5007	* context.
5008	*
5009	* @returns VBox status code.
5010	* @param pVM Pointer to the VM.
5011	* @param pVCpu Pointer to the VMCPU.
5012	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5013	* out-of-sync. Make sure to update the required fields
5014	* before using them.
5015	*
5016	* @remarks No-long-jump zone!!!
5017	*/
5018	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5019	{
5020	RTGCUINTREG uVal = 0;
5021	int rc = VINF_SUCCESS;
5022	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
5023	{
5024	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
5025	pMixedCtx->fs.u64Base = uVal;
5026	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
5027	}
5028	return rc;
5029	}
5030
5031
5032	/**
5033	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5034	* context.
5035	*
5036	* @returns VBox status code.
5037	* @param pVM Pointer to the VM.
5038	* @param pVCpu Pointer to the VMCPU.
5039	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5040	* out-of-sync. Make sure to update the required fields
5041	* before using them.
5042	*
5043	* @remarks No-long-jump zone!!!
5044	*/
5045	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5046	{
5047	RTGCUINTREG uVal = 0;
5048	int rc = VINF_SUCCESS;
5049	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
5050	{
5051	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5052	pMixedCtx->gs.u64Base = uVal;
5053	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
5054	}
5055	return rc;
5056	}
5057
5058
5059	/**
5060	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5061	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5062	*
5063	* @returns VBox status code.
5064	* @param pVM Pointer to the VM.
5065	* @param pVCpu Pointer to the VMCPU.
5066	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5067	* out-of-sync. Make sure to update the required fields
5068	* before using them.
5069	*
5070	* @remarks No-long-jump zone!!!
5071	*/
5072	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5073	{
5074	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5075	return VINF_SUCCESS;
5076
5077	for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5078	{
5079	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5080	pMsr += i;
5081	switch (pMsr->u32IndexMSR)
5082	{
5083	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5084	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5085	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5086	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5087	#if 0
5088	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5089	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5090	#endif
5091	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5092	default:
5093	{
5094	AssertFailed();
5095	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5096	}
5097	}
5098	}
5099	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5100	return VINF_SUCCESS;
5101	}
5102
5103
5104	/**
5105	* Saves the guest control registers from the current VMCS into the guest-CPU
5106	* context.
5107	*
5108	* @returns VBox status code.
5109	* @param pVM Pointer to the VM.
5110	* @param pVCpu Pointer to the VMCPU.
5111	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5112	* out-of-sync. Make sure to update the required fields
5113	* before using them.
5114	*
5115	* @remarks No-long-jump zone!!!
5116	*/
5117	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5118	{
5119	RTGCUINTREG uVal = 0;
5120	RTGCUINTREG uShadow = 0;
5121	int rc = VINF_SUCCESS;
5122
5123	/* Guest CR0. Guest FPU. */
5124	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5125
5126	/* Guest CR4. */
5127	rc \|= hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
5128	AssertRCReturn(rc, rc);
5129
5130	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5131	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5132	{
5133	if ( pVM->hm.s.fNestedPaging
5134	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5135	{
5136	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5137	if (pMixedCtx->cr3 != uVal)
5138	{
5139	CPUMSetGuestCR3(pVCpu, uVal);
5140	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5141	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5142	}
5143
5144	/* We require EFER to check PAE mode. */
5145	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5146
5147	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5148	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5149	{
5150	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5151	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5152	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5153	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5154	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5155	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5156	}
5157	AssertRCReturn(rc, rc);
5158	}
5159	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5160	}
5161	return rc;
5162	}
5163
5164
5165	/**
5166	* Reads a guest segment register from the current VMCS into the guest-CPU
5167	* context.
5168	*
5169	* @returns VBox status code.
5170	* @param pVCpu Pointer to the VMCPU.
5171	* @param idxSel Index of the selector in the VMCS.
5172	* @param idxLimit Index of the segment limit in the VMCS.
5173	* @param idxBase Index of the segment base in the VMCS.
5174	* @param idxAccess Index of the access rights of the segment in the VMCS.
5175	* @param pSelReg Pointer to the segment selector.
5176	*
5177	* @remarks No-long-jump zone!!!
5178	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5179	* as that takes care of whether to read from the VMCS cache or not.
5180	*/
5181	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5182	PCPUMSELREG pSelReg)
5183	{
5184	uint32_t u32Val = 0;
5185	int rc = VMXReadVmcs32(idxSel, &u32Val);
5186	pSelReg->Sel = (uint16_t)u32Val;
5187	pSelReg->ValidSel = (uint16_t)u32Val;
5188	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5189
5190	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5191	pSelReg->u32Limit = u32Val;
5192
5193	RTGCUINTREG uGCVal = 0;
5194	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5195	pSelReg->u64Base = uGCVal;
5196
5197	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5198	pSelReg->Attr.u = u32Val;
5199	AssertRCReturn(rc, rc);
5200
5201	/*
5202	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5203	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5204	*/
5205	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5206	{
5207	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5208	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5209	}
5210	return rc;
5211	}
5212
5213
5214	/**
5215	* Saves the guest segment registers from the current VMCS into the guest-CPU
5216	* context.
5217	*
5218	* @returns VBox status code.
5219	* @param pVM Pointer to the VM.
5220	* @param pVCpu Pointer to the VMCPU.
5221	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5222	* out-of-sync. Make sure to update the required fields
5223	* before using them.
5224	*
5225	* @remarks No-long-jump zone!!!
5226	*/
5227	static int hmR0VmxSaveGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5228	{
5229	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5230	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5231	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5232	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5233	#else
5234	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5235	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5236	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5237	#endif
5238
5239	int rc = VINF_SUCCESS;
5240
5241	/* Guest segment registers. */
5242	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5243	{
5244	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5245	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5246	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5247	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5248	rc \|= VMXLOCAL_READ_SEG(ES, es);
5249	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5250	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5251	AssertRCReturn(rc, rc);
5252
5253	/* Restore segment attributes for real-on-v86 mode hack. */
5254	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5255	{
5256	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5257	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5258	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5259	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5260	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5261	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5262	}
5263	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5264	}
5265
5266	/* Guest LDTR. */
5267	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5268	{
5269	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5270	AssertRCReturn(rc, rc);
5271	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5272	}
5273
5274	/* Guest GDTR. */
5275	RTGCUINTREG uGCVal = 0;
5276	uint32_t u32Val = 0;
5277	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5278	{
5279	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5280	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5281	pMixedCtx->gdtr.pGdt = uGCVal;
5282	pMixedCtx->gdtr.cbGdt = u32Val;
5283	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5284	}
5285
5286	/* Guest IDTR. */
5287	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5288	{
5289	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5290	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5291	pMixedCtx->idtr.pIdt = uGCVal;
5292	pMixedCtx->idtr.cbIdt = u32Val;
5293	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5294	}
5295
5296	/* Guest TR. */
5297	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5298	{
5299	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5300
5301	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5302	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5303	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5304	AssertRCReturn(rc, rc);
5305	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5306	}
5307	return rc;
5308	}
5309
5310
5311	/**
5312	* Saves the guest debug registers from the current VMCS into the guest-CPU
5313	* context.
5314	*
5315	* @returns VBox status code.
5316	* @param pVM Pointer to the VM.
5317	* @param pVCpu Pointer to the VMCPU.
5318	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5319	* out-of-sync. Make sure to update the required fields
5320	* before using them.
5321	*
5322	* @remarks No-long-jump zone!!!
5323	*/
5324	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5325	{
5326	int rc = VINF_SUCCESS;
5327	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5328	{
5329	RTGCUINTREG uVal;
5330	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5331	pMixedCtx->dr[7] = uVal;
5332
5333	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5334	}
5335	return rc;
5336	}
5337
5338
5339	/**
5340	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5341	*
5342	* @returns VBox status code.
5343	* @param pVM Pointer to the VM.
5344	* @param pVCpu Pointer to the VMCPU.
5345	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5346	* out-of-sync. Make sure to update the required fields
5347	* before using them.
5348	*
5349	* @remarks No-long-jump zone!!!
5350	*/
5351	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5352	{
5353	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5354	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5355	return VINF_SUCCESS;
5356	}
5357
5358
5359	/**
5360	* Saves the entire guest state from the currently active VMCS into the
5361	* guest-CPU context. This essentially VMREADs all guest-data.
5362	*
5363	* @returns VBox status code.
5364	* @param pVM Pointer to the VM.
5365	* @param pVCpu Pointer to the VMCPU.
5366	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5367	* out-of-sync. Make sure to update the required fields
5368	* before using them.
5369	*/
5370	static int hmR0VmxSaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5371	{
5372	Assert(pVM);
5373	Assert(pVCpu);
5374	Assert(pMixedCtx);
5375
5376	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5377	return VINF_SUCCESS;
5378
5379	VMMRZCallRing3Disable(pVCpu);
5380
5381	int rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
5382	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGprs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5383
5384	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5385	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5386
5387	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5388	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5389
5390	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx);
5391	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5392
5393	rc = hmR0VmxSaveGuestSysenterMsrs(pVM, pVCpu, pMixedCtx);
5394	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5395
5396	rc = hmR0VmxSaveGuestFSBaseMsr(pVM, pVCpu, pMixedCtx);
5397	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5398
5399	rc = hmR0VmxSaveGuestGSBaseMsr(pVM, pVCpu, pMixedCtx);
5400	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5401
5402	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5403	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5404
5405	rc = hmR0VmxSaveGuestActivityState(pVM, pVCpu, pMixedCtx);
5406	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5407
5408	rc = hmR0VmxSaveGuestApicState(pVM, pVCpu, pMixedCtx);
5409	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5410
5411	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5412	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5413
5414	VMMRZCallRing3Enable(pVCpu);
5415	return rc;
5416	}
5417
5418
5419	/**
5420	* Check per-VM and per-VCPU force flag actions that require us to go back to
5421	* ring-3 for one reason or another.
5422	*
5423	* @returns VBox status code (information status code included).
5424	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5425	* ring-3.
5426	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5427	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5428	* interrupts)
5429	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5430	* all EMTs to be in ring-3.
5431	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5432	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5433	* to the EM loop.
5434	*
5435	* @param pVM Pointer to the VM.
5436	* @param pVCpu Pointer to the VMCPU.
5437	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5438	* out-of-sync. Make sure to update the required fields
5439	* before using them.
5440	*/
5441	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5442	{
5443	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5444
5445	int rc = VERR_INTERNAL_ERROR_5;
5446	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5447	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5448	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5449	{
5450	/* We need the control registers now, make sure the guest-CPU context is updated. */
5451	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5452	AssertRCReturn(rc, rc);
5453
5454	/* Pending HM CR3 sync. */
5455	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5456	{
5457	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5458	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5459	}
5460	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5461	{
5462	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5463	AssertRC(rc);
5464	}
5465
5466	/* Pending PGM C3 sync. */
5467	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5468	{
5469	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5470	if (rc != VINF_SUCCESS)
5471	{
5472	AssertRC(rc);
5473	Log2(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5474	return rc;
5475	}
5476	}
5477
5478	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5479	/* -XXX- what was that about single stepping? */
5480	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5481	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5482	{
5483	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5484	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5485	Log2(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5486	return rc;
5487	}
5488
5489	/* Pending VM request packets, such as hardware interrupts. */
5490	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5491	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5492	{
5493	Log2(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5494	return VINF_EM_PENDING_REQUEST;
5495	}
5496
5497	/* Pending PGM pool flushes. */
5498	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5499	{
5500	Log2(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5501	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5502	}
5503
5504	/* Pending DMA requests. */
5505	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5506	{
5507	Log2(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5508	return VINF_EM_RAW_TO_R3;
5509	}
5510	}
5511
5512	/* Paranoia. */
5513	Assert(rc != VERR_EM_INTERPRETER);
5514	return VINF_SUCCESS;
5515	}
5516
5517
5518	/**
5519	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5520	* VT-x to execute any instruction.
5521	*
5522	* @param pvCpu Pointer to the VMCPU.
5523	*/
5524	static void hmR0VmxUpdateTRPMTrap(PVMCPU pVCpu)
5525	{
5526	if (pVCpu->hm.s.Event.fPending)
5527	{
5528	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5529	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5530	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5531	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5532
5533	/* If a trap was already pending, we did something wrong! */
5534	Assert(TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP);
5535
5536	/* A page-fault exception during a page-fault would become a double-fault. */
5537	AssertMsg(uVectorType != VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT \|\| uVector != X86_XCPT_PF,
5538	("%#RX64 uVectorType=%#x uVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uVectorType, uVector));
5539
5540	TRPMEVENT enmTrapType;
5541	switch (uVectorType)
5542	{
5543	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5544	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5545	enmTrapType = TRPM_HARDWARE_INT;
5546	break;
5547	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5548	enmTrapType = TRPM_SOFTWARE_INT;
5549	break;
5550	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5551	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5552	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5553	enmTrapType = TRPM_TRAP;
5554	break;
5555	default:
5556	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5557	enmTrapType = TRPM_32BIT_HACK;
5558	break;
5559	}
5560	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5561	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5562	AssertRC(rc);
5563	if (fErrorCodeValid)
5564	TRPMSetErrorCode(pVCpu, uErrorCode);
5565	AssertRC(rc);
5566	pVCpu->hm.s.Event.fPending = false;
5567	}
5568
5569	/* Clear the VT-x state bits to prevent any stale injection. */
5570	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
5571	AssertRC(rc2);
5572	}
5573
5574
5575	/**
5576	* Does the necessary state syncing before doing a longjmp to ring-3.
5577	*
5578	* @param pVM Pointer to the VM.
5579	* @param pVCpu Pointer to the VMCPU.
5580	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5581	* out-of-sync. Make sure to update the required fields
5582	* before using them.
5583	* @param rcExit The reason for exiting to ring-3. Can be
5584	* VINF_VMM_UNKNOWN_RING3_CALL.
5585	*
5586	* @remarks No-long-jmp zone!!!
5587	*/
5588	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5589	{
5590	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5591
5592	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
5593	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL);
5594	AssertRC(rc);
5595
5596	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5597	if (CPUMIsGuestFPUStateActive(pVCpu))
5598	{
5599	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5600	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5601	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5602	}
5603
5604	/* Restore debug registers if necessary and resync on next R0 reentry. */
5605	if (CPUMIsGuestDebugStateActive(pVCpu))
5606	{
5607	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5608	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5609	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5610	}
5611	else if (CPUMIsHyperDebugStateActive(pVCpu))
5612	{
5613	CPUMR0LoadHostDebugState(pVM, pVCpu);
5614	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5615	}
5616
5617	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5618	}
5619
5620
5621	/**
5622	* An action requires us to go back to ring-3. This function does the necessary
5623	* steps before we can safely return to ring-3. This is not the same as longjmps
5624	* to ring-3, this is voluntary.
5625	*
5626	* @param pVM Pointer to the VM.
5627	* @param pVCpu Pointer to the VMCPU.
5628	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5629	* out-of-sync. Make sure to update the required fields
5630	* before using them.
5631	* @param rcExit The reason for exiting to ring-3. Can be
5632	* VINF_VMM_UNKNOWN_RING3_CALL.
5633	*/
5634	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5635	{
5636	Assert(pVM);
5637	Assert(pVCpu);
5638	Assert(pMixedCtx);
5639	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5640
5641	/* We want to see what the guest-state was before VM-entry, don't resync here, as we will never continue guest execution.*/
5642	if (rcExit == VERR_VMX_INVALID_GUEST_STATE)
5643	return;
5644
5645	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5646	VMMRZCallRing3Disable(pVCpu);
5647	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5648
5649	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5650	hmR0VmxUpdateTRPMTrap(pVCpu);
5651
5652	/* Sync. the guest state. */
5653	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5654	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5655
5656	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5657	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5658	\| CPUM_CHANGED_LDTR
5659	\| CPUM_CHANGED_GDTR
5660	\| CPUM_CHANGED_IDTR
5661	\| CPUM_CHANGED_TR
5662	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5663
5664	/* On our way back from ring-3 the following needs to be done. */
5665	/** @todo This can change with preemption hooks. */
5666	if (rcExit == VINF_EM_RAW_INTERRUPT)
5667	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5668	else
5669	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5670
5671	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5672	VMMRZCallRing3Enable(pVCpu);
5673	}
5674
5675
5676	/**
5677	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5678	* longjump to ring-3 and possibly get preempted.
5679	*
5680	* @param pVCpu Pointer to the VMCPU.
5681	* @param enmOperation The operation causing the ring-3 longjump.
5682	* @param pvUser The user argument (pointer to the possibly
5683	* out-of-date guest-CPU context).
5684	*
5685	* @remarks Must never be called with @a enmOperation ==
5686	* VMMCALLRING3_VM_R0_ASSERTION.
5687	*/
5688	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5689	{
5690	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5691	Assert(pVCpu);
5692	Assert(pvUser);
5693	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5694	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5695
5696	VMMRZCallRing3Disable(pVCpu);
5697	Log(("hmR0VmxLongJmpToRing3\n"));
5698	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5699	VMMRZCallRing3Enable(pVCpu);
5700	}
5701
5702
5703	/**
5704	* Injects any pending TRPM trap into the VM by updating the VMCS.
5705	*
5706	* @returns VBox status code (informational status code included).
5707	* @param pVM Pointer to the VM.
5708	* @param pVCpu Pointer to the VMCPU.
5709	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5710	* out-of-sync. Make sure to update the required fields
5711	* before using them.
5712	*/
5713	static int hmR0VmxInjectTRPMTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5714	{
5715	if (!TRPMHasTrap(pVCpu))
5716	return VINF_SUCCESS;
5717
5718	uint8_t u8Vector = 0;
5719	TRPMEVENT enmTrpmEvent = TRPM_SOFTWARE_INT;
5720	RTGCUINT uErrCode = 0;
5721
5722	int rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmTrpmEvent, &uErrCode, NULL /* puCr2 */);
5723	AssertRCReturn(rc, rc);
5724	Assert(enmTrpmEvent != TRPM_SOFTWARE_INT);
5725
5726	rc = TRPMResetTrap(pVCpu);
5727	AssertRCReturn(rc, rc);
5728
5729	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5730	uint32_t u32IntrInfo = u8Vector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5731	if (enmTrpmEvent == TRPM_TRAP)
5732	{
5733	switch (u8Vector)
5734	{
5735	case X86_XCPT_BP:
5736	case X86_XCPT_OF:
5737	{
5738	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5739	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5740	break;
5741	}
5742
5743	case X86_XCPT_DF:
5744	case X86_XCPT_TS:
5745	case X86_XCPT_NP:
5746	case X86_XCPT_SS:
5747	case X86_XCPT_GP:
5748	case X86_XCPT_PF:
5749	case X86_XCPT_AC:
5750	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5751	them which VT-x/VMM pushes to the guest stack. */
5752	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5753	/* no break! */
5754	default:
5755	{
5756	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5757	break;
5758	}
5759	}
5760	}
5761	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5762	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5763	else
5764	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5765
5766	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5767	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, uErrCode);
5768	}
5769
5770
5771	/**
5772	* Checks if there are any pending guest interrupts to be delivered and injects
5773	* them into the VM by updating the VMCS.
5774	*
5775	* @returns VBox status code (informational status codes included).
5776	* @param pVM Pointer to the VM.
5777	* @param pVCpu Pointer to the VMCPU.
5778	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5779	* out-of-sync. Make sure to update the required fields
5780	* before using them.
5781	*
5782	* @remarks Must be called after hmR0VmxLoadGuestIntrState().
5783	*/
5784	static int hmR0VmxInjectPendingInterrupt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5785	{
5786	/* First inject any pending HM interrupts. */
5787	if (pVCpu->hm.s.Event.fPending)
5788	{
5789	Log(("Pending event\n"));
5790	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, 0 /* cbInstr */,
5791	pVCpu->hm.s.Event.u32ErrCode);
5792	AssertRCReturn(rc, rc);
5793	pVCpu->hm.s.Event.fPending = false;
5794	return rc;
5795	}
5796
5797	/** @todo SMI. SMIs take priority over NMIs. */
5798
5799	/* NMI. NMIs take priority over regular interrupts . */
5800	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
5801	{
5802	/* Construct an NMI interrupt and inject it into the VMCS. */
5803	RTGCUINTPTR uIntrInfo;
5804	uIntrInfo = X86_XCPT_NMI;
5805	uIntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5806	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5807	Log(("Injecting NMI\n"));
5808	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5809	AssertRCReturn(rc, rc);
5810	return rc;
5811	}
5812
5813	/* We need the guests's RFLAGS for sure from this point on, make sure it is updated. */
5814	int rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
5815	AssertRCReturn(rc, rc);
5816
5817	/* If there isn't any active trap, check if we have pending interrupts and convert them to TRPM traps and deliver them. */
5818	if (!TRPMHasTrap(pVCpu))
5819	{
5820	/* Check if there are guest external interrupts (PIC/APIC) pending. */
5821	if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5822	{
5823	/*
5824	* If the guest can receive interrupts now (interrupts enabled and no interrupt inhibition is active) convert
5825	* the PDM interrupt into a TRPM event and inject it.
5826	*/
5827	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5828	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5829	{
5830	uint8_t u8Interrupt = 0;
5831	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5832	if (RT_SUCCESS(rc))
5833	{
5834	Log(("PDMGetInterrupt: u8Interrupt=%#x\n", u8Interrupt));
5835	/* Convert pending interrupt from PIC/APIC into TRPM and handle it below. */
5836	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
5837	AssertRCReturn(rc, rc);
5838	}
5839	else
5840	{
5841	/** @todo Does this actually happen? If not turn it into an assertion. */
5842	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5843	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5844	}
5845	}
5846	else if ( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT)
5847	&& (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5848	{
5849	/* Instruct VT-x to cause an interrupt-window exit as soon as the guest is ready to receive interrupts again. */
5850	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5851	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5852	AssertRCReturn(rc, rc);
5853	}
5854	/* else we will deliver interrupts whenever the guest exits next and it's in a state to receive interrupts. */
5855	}
5856	}
5857
5858	/* If interrupts can be delivered, inject it into the VM. */
5859	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5860	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
5861	&& TRPMHasTrap(pVCpu))
5862	{
5863	Log(("Injecting TRPM trap\n"));
5864	rc = hmR0VmxInjectTRPMTrap(pVM, pVCpu, pMixedCtx);
5865	Assert(!TRPMHasTrap(pVCpu));
5866	AssertRCReturn(rc, rc);
5867	}
5868	return rc;
5869	}
5870
5871	/**
5872	* Injects an invalid-opcode (#UD) exception into the VM.
5873	*
5874	* @returns VBox status code (informational status code included).
5875	* @param pVM Pointer to the VM.
5876	* @param pVCpu Pointer to the VMCPU.
5877	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5878	* out-of-sync. Make sure to update the required fields
5879	* before using them.
5880	*/
5881	DECLINLINE(int) hmR0VmxInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5882	{
5883	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5884	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5885	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5886	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5887	}
5888
5889
5890	/**
5891	* Injects a double-fault (#DF) exception into the VM.
5892	*
5893	* @returns VBox status code (informational status code included).
5894	* @param pVM Pointer to the VM.
5895	* @param pVCpu Pointer to the VMCPU.
5896	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5897	* out-of-sync. Make sure to update the required fields
5898	* before using them.
5899	*/
5900	DECLINLINE(int) hmR0VmxInjectXcptDF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5901	{
5902	/* Inject the double-fault. */
5903	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5904	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5905	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5906	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5907	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5908	}
5909
5910
5911	/**
5912	* Injects a debug (#DB) exception into the VM.
5913	*
5914	* @returns VBox status code (informational status code included).
5915	* @param pVM Pointer to the VM.
5916	* @param pVCpu Pointer to the VMCPU.
5917	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5918	* out-of-sync. Make sure to update the required fields
5919	* before using them.
5920	*/
5921	DECLINLINE(int) hmR0VmxInjectXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5922	{
5923	/* Inject the debug-exception. */
5924	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5925	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5926	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5927	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5928	}
5929
5930
5931	/**
5932	* Injects a overflow (#OF) exception into the VM.
5933	*
5934	* @returns VBox status code (informational status code included).
5935	* @param pVM Pointer to the VM.
5936	* @param pVCpu Pointer to the VMCPU.
5937	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5938	* out-of-sync. Make sure to update the required fields
5939	* before using them.
5940	* @param cbInstr The value of RIP that is to be pushed on the guest
5941	* stack.
5942	*/
5943	DECLINLINE(int) hmR0VmxInjectXcptOF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5944	{
5945	/* Inject the overflow exception. */
5946	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5947	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5948	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5949	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5950	}
5951
5952
5953	/**
5954	* Injects a general-protection (#GP) fault into the VM.
5955	*
5956	* @returns VBox status code (informational status code included).
5957	* @param pVM Pointer to the VM.
5958	* @param pVCpu Pointer to the VMCPU.
5959	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5960	* out-of-sync. Make sure to update the required fields
5961	* before using them.
5962	* @param u32ErrorCode The error code associated with the #GP.
5963	*/
5964	DECLINLINE(int) hmR0VmxInjectXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode)
5965	{
5966	/* Inject the general-protection fault. */
5967	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5968	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5969	if (fErrorCodeValid)
5970	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5971	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5972	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode);
5973	}
5974
5975
5976	/**
5977	* Injects a software interrupt (INTn) into the VM.
5978	*
5979	* @returns VBox status code (informational status code included).
5980	* @param pVM Pointer to the VM.
5981	* @param pVCpu Pointer to the VMCPU.
5982	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5983	* out-of-sync. Make sure to update the required fields
5984	* before using them.
5985	* @param uVector The software interrupt vector number.
5986	* @param cbInstr The value of RIP that is to be pushed on the guest
5987	* stack.
5988	*/
5989	DECLINLINE(int) hmR0VmxInjectIntN(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
5990	{
5991	/* Inject the INTn. */
5992	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5993	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5994	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5995	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5996	}
5997
5998
5999	/**
6000	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6001	* stack.
6002	*
6003	* @returns VBox status code (information status code included).
6004	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6005	* @param pVM Pointer to the VM.
6006	* @param pMixedCtx Pointer to the guest-CPU context.
6007	* @param uValue The value to push to the guest stack.
6008	*/
6009	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6010	{
6011	/*
6012	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6013	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6014	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6015	*/
6016	if (pMixedCtx->sp == 1)
6017	return VINF_EM_RESET;
6018	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6019	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6020	AssertRCReturn(rc, rc);
6021	return rc;
6022	}
6023
6024
6025	/**
6026	* Injects an event into the guest upon VM-entry by updating the relevant fields
6027	* in the VM-entry area in the VMCS.
6028	*
6029	* @returns VBox status code (informational error codes included).
6030	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6031	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6032	*
6033	* @param pVM Pointer to the VM.
6034	* @param pVCpu Pointer to the VMCPU.
6035	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6036	* out-of-sync. Make sure to update the required fields
6037	* before using them.
6038	* @param u64IntrInfo The VM-entry interruption-information field.
6039	* @param cbInstr The VM-entry instruction length in bytes (for software
6040	* interrupts, exceptions and privileged software
6041	* exceptions).
6042	* @param u32ErrCode The VM-entry exception error code.
6043	*
6044	* @remarks No-long-jump zone!!!
6045	*/
6046	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6047	uint32_t u32ErrCode)
6048	{
6049	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6050	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6051	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6052
6053	/* We require CR0 to check if the guest is in real-mode. */
6054	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6055	AssertRCReturn(rc, rc);
6056
6057	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6058	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6059
6060	/*
6061	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6062	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6063	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6064	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6065	*/
6066	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6067	{
6068	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6069	{
6070	Assert(PDMVmmDevHeapIsEnabled(pVM));
6071	Assert(pVM->hm.s.vmx.pRealModeTSS);
6072
6073	/* Save the required guest state bits from the VMCS. */
6074	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
6075	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
6076	AssertRCReturn(rc, rc);
6077
6078	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6079	const size_t cbIdtEntry = 4;
6080	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6081	{
6082	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6083	if (uVector == X86_XCPT_DF)
6084	return VINF_EM_RESET;
6085	else if (uVector == X86_XCPT_GP)
6086	{
6087	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6088	return hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
6089	}
6090
6091	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6092	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6093	return hmR0VmxInjectXcptGP(pVM, pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */);
6094	}
6095
6096	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6097	uint16_t uGuestIp = pMixedCtx->ip;
6098	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6099	{
6100	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6101	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6102	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6103	}
6104	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6105	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6106
6107	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6108	uint16_t offIdtEntry = 0;
6109	RTSEL selIdtEntry = 0;
6110	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6111	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6112	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6113	AssertRCReturn(rc, rc);
6114
6115	/* Construct the stack frame for the interrupt/exception handler. */
6116	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6117	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6118	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6119	AssertRCReturn(rc, rc);
6120
6121	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6122	if (rc == VINF_SUCCESS)
6123	{
6124	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6125	pMixedCtx->rip = offIdtEntry;
6126	pMixedCtx->cs.Sel = selIdtEntry;
6127	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6128	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6129	\| HM_CHANGED_GUEST_RIP
6130	\| HM_CHANGED_GUEST_RFLAGS
6131	\| HM_CHANGED_GUEST_RSP;
6132	}
6133	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6134	return rc;
6135	}
6136	else
6137	{
6138	/*
6139	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6140	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6141	*/
6142	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6143	}
6144	}
6145
6146	/* Add the valid bit, maybe the caller was lazy. */
6147	u32IntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6148
6149	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6150	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6151	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6152
6153	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6154	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6155	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6156	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6157	AssertRCReturn(rc, rc);
6158	return rc;
6159	}
6160
6161
6162	/**
6163	* Enters the VT-x session.
6164	*
6165	* @returns VBox status code.
6166	* @param pVM Pointer to the VM.
6167	* @param pVCpu Pointer to the VMCPU.
6168	* @param pCpu Pointer to the CPU info struct.
6169	*/
6170	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6171	{
6172	AssertPtr(pVM);
6173	AssertPtr(pVCpu);
6174	Assert(pVM->hm.s.vmx.fSupported);
6175	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6176	NOREF(pCpu);
6177
6178	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6179
6180	/* Make sure we're in VMX root mode. */
6181	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6182	if (!(u32HostCR4 & X86_CR4_VMXE))
6183	{
6184	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6185	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6186	}
6187
6188	/* Load the active VMCS as the current one. */
6189	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6190	if (RT_FAILURE(rc))
6191	return rc;
6192
6193	/** @todo this will change with preemption hooks where can can VMRESUME as long
6194	* as we're no preempted. */
6195	pVCpu->hm.s.fResumeVM = false;
6196	return VINF_SUCCESS;
6197	}
6198
6199
6200	/**
6201	* Leaves the VT-x session.
6202	*
6203	* @returns VBox status code.
6204	* @param pVM Pointer to the VM.
6205	* @param pVCpu Pointer to the VMCPU.
6206	* @param pCtx Pointer to the guest-CPU context.
6207	*/
6208	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6209	{
6210	AssertPtr(pVCpu);
6211	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6212	NOREF(pVM);
6213	NOREF(pCtx);
6214
6215	/** @todo this will change with preemption hooks where we only VMCLEAR when
6216	* we are actually going to be preempted, not all the time like we
6217	* currently do. */
6218	/*
6219	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6220	* and mark the VMCS launch-state as "clear".
6221	*/
6222	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6223	return rc;
6224	}
6225
6226
6227	/**
6228	* Saves the host state in the VMCS host-state.
6229	* Sets up the VM-exit MSR-load area.
6230	*
6231	* The CPU state will be loaded from these fields on every successful VM-exit.
6232	*
6233	* @returns VBox status code.
6234	* @param pVM Pointer to the VM.
6235	* @param pVCpu Pointer to the VMCPU.
6236	*
6237	* @remarks No-long-jump zone!!!
6238	*/
6239	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6240	{
6241	AssertPtr(pVM);
6242	AssertPtr(pVCpu);
6243	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6244
6245	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6246
6247	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6248	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6249	return VINF_SUCCESS;
6250
6251	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6252	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6253
6254	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6255	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6256
6257	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6258	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6259
6260	/* Reset the host-state-changed flag. */
6261	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6262	return rc;
6263	}
6264
6265
6266	/**
6267	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6268	* loaded from these fields on every successful VM-entry.
6269	*
6270	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6271	* Sets up the VM-entry controls.
6272	* Sets up the appropriate VMX non-root function to execute guest code based on
6273	* the guest CPU mode.
6274	*
6275	* @returns VBox status code.
6276	* @param pVM Pointer to the VM.
6277	* @param pVCpu Pointer to the VMCPU.
6278	* @param pCtx Pointer to the guest-CPU context.
6279	*
6280	* @remarks No-long-jump zone!!!
6281	*/
6282	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6283	{
6284	AssertPtr(pVM);
6285	AssertPtr(pVCpu);
6286	AssertPtr(pCtx);
6287	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6288
6289	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6290
6291	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6292
6293	/* Determine real-on-v86 mode. */
6294	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6295	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6296	&& CPUMIsGuestInRealModeEx(pCtx))
6297	{
6298	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6299	}
6300
6301	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6302	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6303
6304	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu, pCtx);
6305	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6306
6307	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6308	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6309
6310	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6311	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6312
6313	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6314	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6315
6316	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6317	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6318
6319	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6320	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6321
6322	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6323	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6324
6325	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6326	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6327
6328	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6329	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6330
6331	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6332	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6333	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6334
6335	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6336	return rc;
6337	}
6338
6339
6340	/**
6341	* Does the preparations before executing guest code in VT-x.
6342	*
6343	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6344	* recompiler. We must be cautious what we do here regarding committing
6345	* guest-state information into the the VMCS assuming we assuredly execute the
6346	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6347	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6348	* that the recompiler can (and should) use them when it resumes guest
6349	* execution. Otherwise such operations must be done when we can no longer
6350	* exit to ring-3.
6351	*
6352	* @returns VBox status code (informational status codes included).
6353	* @retval VINF_SUCCESS if we can proceed with running the guest.
6354	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6355	* into the guest.
6356	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6357	*
6358	* @param pVM Pointer to the VM.
6359	* @param pVCpu Pointer to the VMCPU.
6360	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6361	* out-of-sync. Make sure to update the required fields
6362	* before using them.
6363	* @param pVmxTransient Pointer to the VMX transient structure.
6364	*
6365	* @remarks Called with preemption disabled.
6366	*/
6367	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6368	{
6369	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6370
6371	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6372	PGMRZDynMapFlushAutoSet(pVCpu);
6373	#endif
6374
6375	/* Check force flag actions that might require us to go back to ring-3. */
6376	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6377	if (rc != VINF_SUCCESS)
6378	return rc;
6379
6380	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6381	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6382	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6383	{
6384	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6385	RTGCPHYS GCPhysApicBase;
6386	GCPhysApicBase = pMixedCtx->msrApicBase;
6387	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6388
6389	/* Unalias any existing mapping. */
6390	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6391	AssertRCReturn(rc, rc);
6392
6393	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6394	Log2(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6395	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6396	AssertRCReturn(rc, rc);
6397
6398	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6399	}
6400
6401	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6402	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6403	pVmxTransient->uEFlags = ASMIntDisableFlags();
6404	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6405	{
6406	ASMSetFlags(pVmxTransient->uEFlags);
6407	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6408	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6409	return VINF_EM_RAW_INTERRUPT;
6410	}
6411	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6412	#endif
6413
6414	/*
6415	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6416	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6417	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6418	*/
6419	hmR0VmxLoadGuestIntrState(pVM, pVCpu, pMixedCtx);
6420	rc = hmR0VmxInjectPendingInterrupt(pVM, pVCpu, pMixedCtx);
6421	AssertRCReturn(rc, rc);
6422	return rc;
6423	}
6424
6425
6426	/**
6427	* Prepares to run guest code in VT-x and we've committed to doing so. This
6428	* means there is no backing out to ring-3 or anywhere else at this
6429	* point.
6430	*
6431	* @param pVM Pointer to the VM.
6432	* @param pVCpu Pointer to the VMCPU.
6433	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6434	* out-of-sync. Make sure to update the required fields
6435	* before using them.
6436	* @param pVmxTransient Pointer to the VMX transient structure.
6437	*
6438	* @remarks Called with preemption disabled.
6439	* @remarks No-long-jump zone!!!
6440	*/
6441	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6442	{
6443	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6444	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6445
6446	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6447	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6448	pVmxTransient->uEFlags = ASMIntDisableFlags();
6449	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6450	#endif
6451
6452	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6453	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6454	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6455	int rc = VINF_SUCCESS;
6456	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6457	{
6458	rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pMixedCtx);
6459	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6460	}
6461	else if (pVCpu->hm.s.fContextUseFlags)
6462	{
6463	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6464	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6465	}
6466	AssertRC(rc);
6467	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6468
6469	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6470	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6471	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6472
6473	Assert(pVM->hm.s.vmx.pfnFlushTaggedTlb);
6474	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6475	pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu); /* Flush the TLB of guest entries as necessary. */
6476
6477	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6478	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6479	{
6480	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6481	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6482	}
6483
6484	/*
6485	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6486	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6487	*/
6488	if (pVM->hm.s.fTPRPatchingActive)
6489	{
6490	Assert(!CPUMIsGuestInLongMode(pVCpu));
6491
6492	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6493	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6494	AssertRC(rc);
6495
6496	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6497	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6498	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6499	see hmR0VmxLoadGuestApicState(). */
6500	}
6501
6502	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6503	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6504	to start executing. */
6505	}
6506
6507
6508	/**
6509	* Performs some essential restoration of state after running guest code in
6510	* VT-x.
6511	*
6512	* @param pVM Pointer to the VM.
6513	* @param pVCpu Pointer to the VMCPU.
6514	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6515	* out-of-sync. Make sure to update the required fields
6516	* before using them.
6517	* @param pVmxTransient Pointer to the VMX transient structure.
6518	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6519	*
6520	* @remarks Called with interrupts disabled.
6521	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6522	* unconditionally when it is safe to do so.
6523	*/
6524	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6525	{
6526	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6527	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6528
6529	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6530	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6531	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6532	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6533	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6534
6535	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6536	{
6537	/** @todo Find a way to fix hardcoding a guestimate. */
6538	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6539	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6540	}
6541
6542	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6543	Assert(!(ASMGetFlags() & X86_EFL_IF));
6544	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6545
6546	/* Restore the effects of TPR patching if any. */
6547	if (pVM->hm.s.fTPRPatchingActive)
6548	{
6549	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6550	AssertRC(rc);
6551	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6552	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6553	}
6554
6555	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6556	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6557
6558	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6559	uint32_t uExitReason;
6560	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6561	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6562	AssertRC(rc);
6563	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6564	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6565
6566	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6567	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6568
6569	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6570	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6571	{
6572	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6573	return;
6574	}
6575
6576	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6577	{
6578	/* Update the guest interruptibility-state from the VMCS. */
6579	hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
6580
6581	/*
6582	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6583	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6584	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6585	*/
6586	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6587	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6588	{
6589	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6590	AssertRC(rc);
6591	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6592	}
6593	}
6594	}
6595
6596
6597	/**
6598	* Runs the guest code using VT-x.
6599	*
6600	* @returns VBox status code.
6601	* @param pVM Pointer to the VM.
6602	* @param pVCpu Pointer to the VMCPU.
6603	* @param pCtx Pointer to the guest-CPU context.
6604	*
6605	* @remarks Called with preemption disabled.
6606	*/
6607	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6608	{
6609	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6610	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6611
6612	VMXTRANSIENT VmxTransient;
6613	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6614	int rc = VERR_INTERNAL_ERROR_5;
6615	unsigned cLoops = 0;
6616
6617	for (;; cLoops++)
6618	{
6619	Assert(!HMR0SuspendPending());
6620	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6621	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6622	(unsigned)RTMpCpuId(), cLoops));
6623
6624	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6625	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6626	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6627	if (rc != VINF_SUCCESS)
6628	break;
6629
6630	/*
6631	* No longjmps to ring-3 from this point on!!!
6632	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6633	* This also disables flushing of the R0-logger instance (if any).
6634	*/
6635	VMMRZCallRing3Disable(pVCpu);
6636	VMMRZCallRing3RemoveNotification(pVCpu);
6637	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6638
6639	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6640	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6641
6642	/*
6643	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6644	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6645	*/
6646	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6647	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6648	{
6649	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6650	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6651	return rc;
6652	}
6653
6654	/* Handle the VM-exit. */
6655	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6656	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6657	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6658	rc = (*s_apfnVMExitHandlers[VmxTransient.uExitReason])(pVM, pVCpu, pCtx, &VmxTransient);
6659	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6660	if (rc != VINF_SUCCESS)
6661	break;
6662	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6663	{
6664	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6665	rc = VINF_EM_RAW_INTERRUPT;
6666	break;
6667	}
6668	}
6669
6670	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6671	if (rc == VERR_EM_INTERPRETER)
6672	rc = VINF_EM_RAW_EMULATE_INSTR;
6673	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6674	return rc;
6675	}
6676
6677	#if 0
6678	DECLINLINE(int) hmR0VmxHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, unsigned rcReason)
6679	{
6680	int rc;
6681	switch (rcReason)
6682	{
6683	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6684	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6685	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6686	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6687	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6688	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6689	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6690	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6691	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6692	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6693	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6694	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6695	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6696	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6697	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6698	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6699	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6700	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6701	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6702	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6703	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6704	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6705	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6706	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6707	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6708	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6709	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6710	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6711	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6712	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6713	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6714	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6715	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6716
6717	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6718	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6719	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6720	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6721	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6722	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6723	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6724	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6725	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6726
6727	case VMX_EXIT_VMCALL:
6728	case VMX_EXIT_VMCLEAR:
6729	case VMX_EXIT_VMLAUNCH:
6730	case VMX_EXIT_VMPTRLD:
6731	case VMX_EXIT_VMPTRST:
6732	case VMX_EXIT_VMREAD:
6733	case VMX_EXIT_VMRESUME:
6734	case VMX_EXIT_VMWRITE:
6735	case VMX_EXIT_VMXOFF:
6736	case VMX_EXIT_VMXON:
6737	case VMX_EXIT_INVEPT:
6738	case VMX_EXIT_INVVPID:
6739	case VMX_EXIT_VMFUNC:
6740	rc = hmR0VmxExitInjectXcptUD(pVM, pVCpu, pMixedCtx, pVmxTransient);
6741	break;
6742	default:
6743	rc = hmR0VmxExitErrUndefined(pVM, pVCpu, pMixedCtx, pVmxTransient);
6744	break;
6745	}
6746	return rc;
6747	}
6748	#endif
6749
6750	#ifdef DEBUG
6751	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6752	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6753	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6754	# define VMX_ASSERT_PREEMPT_CPUID() \
6755	do \
6756	{ \
6757	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6758	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6759	} while (0)
6760
6761	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6762	do { \
6763	AssertPtr(pVM); \
6764	AssertPtr(pVCpu); \
6765	AssertPtr(pMixedCtx); \
6766	AssertPtr(pVmxTransient); \
6767	Assert(pVmxTransient->fVMEntryFailed == false); \
6768	Assert(ASMIntAreEnabled()); \
6769	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6770	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6771	LogFunc(("\n")); \
6772	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6773	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6774	VMX_ASSERT_PREEMPT_CPUID(); \
6775	} while (0)
6776	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6777	do { \
6778	LogFunc(("\n")); \
6779	} while(0)
6780	#else /* Release builds */
6781	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6782	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6783	#endif
6784
6785
6786	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6787	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6788	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6789	/**
6790	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6791	*/
6792	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6793	{
6794	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6795	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6796	return VINF_SUCCESS;
6797	}
6798
6799
6800	/**
6801	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6802	*/
6803	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6804	{
6805	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6806	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6807	AssertRCReturn(rc, rc);
6808
6809	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6810	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6811	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6812
6813	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6814	return VINF_EM_RAW_INTERRUPT;
6815
6816	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6817	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
6818	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6819	return VINF_SUCCESS;
6820	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6821	return rc;
6822
6823	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6824	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6825	switch (uIntrType)
6826	{
6827	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6828	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6829	/* no break */
6830	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
6831	{
6832	switch (uVector)
6833	{
6834	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6835	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6836	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6837	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6838	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6839	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6840	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
6841	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6842	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6843	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6844	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6845	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
6846	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6847	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
6848	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6849	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
6850	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6851	#endif
6852	default:
6853	{
6854	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6855	AssertRCReturn(rc, rc);
6856
6857	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
6858	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6859	{
6860	Assert(pVM->hm.s.vmx.pRealModeTSS);
6861	Assert(PDMVmmDevHeapIsEnabled(pVM));
6862	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6863	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
6864	AssertRCReturn(rc, rc);
6865	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
6866	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
6867	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
6868	AssertRCReturn(rc, rc);
6869	}
6870	else
6871	{
6872	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
6873	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
6874	}
6875	break;
6876	}
6877	}
6878	break;
6879	}
6880
6881	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
6882	default:
6883	{
6884	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
6885	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
6886	break;
6887	}
6888	}
6889	return rc;
6890	}
6891
6892
6893	/**
6894	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
6895	*/
6896	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6897	{
6898	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6899
6900	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
6901	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
6902	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
6903	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
6904	AssertRCReturn(rc, rc);
6905
6906	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and resume guest execution. */
6907	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6908	return VINF_SUCCESS;
6909	}
6910
6911
6912	/**
6913	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
6914	*/
6915	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6916	{
6917	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6918	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
6919	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6920	}
6921
6922
6923	/**
6924	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
6925	*/
6926	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6927	{
6928	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6929	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6930	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6931	AssertRCReturn(rc, rc);
6932
6933	pMixedCtx->rip += pVmxTransient->cbInstr;
6934	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6935
6936	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6937	return VINF_SUCCESS;
6938	}
6939
6940
6941	/**
6942	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
6943	*/
6944	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6945	{
6946	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6947	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6948	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6949	AssertRCReturn(rc, rc);
6950
6951	pMixedCtx->rip += pVmxTransient->cbInstr;
6952	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6953
6954	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6955	return VINF_SUCCESS;
6956	}
6957
6958
6959	/**
6960	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
6961	*/
6962	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6963	{
6964	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6965	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6966	if (RT_LIKELY(rc == VINF_SUCCESS))
6967	{
6968	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6969	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6970	AssertRCReturn(rc, rc);
6971	Assert(pVmxTransient->cbInstr == 2);
6972
6973	Log(("hmR0VmxExitCpuid: RIP=%#RX64\n", pMixedCtx->rip));
6974	pMixedCtx->rip += pVmxTransient->cbInstr;
6975	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6976	}
6977	else
6978	{
6979	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6980	rc = VERR_EM_INTERPRETER;
6981	}
6982	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6983	return rc;
6984	}
6985
6986
6987	/**
6988	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
6989	*/
6990	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6991	{
6992	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6993	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
6994	AssertRCReturn(rc, rc);
6995
6996	if (pMixedCtx->cr4 & X86_CR4_SMXE)
6997	return VINF_EM_RAW_EMULATE_INSTR;
6998
6999	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7000	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7001	}
7002
7003
7004	/**
7005	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7006	*/
7007	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7008	{
7009	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7010	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7011	AssertRCReturn(rc, rc);
7012
7013	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7014	if (RT_LIKELY(rc == VINF_SUCCESS))
7015	{
7016	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7017	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7018	AssertRCReturn(rc, rc);
7019	Assert(pVmxTransient->cbInstr == 2);
7020
7021	pMixedCtx->rip += pVmxTransient->cbInstr;
7022	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7023
7024	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7025	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7026	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7027	}
7028	else
7029	{
7030	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7031	rc = VERR_EM_INTERPRETER;
7032	}
7033	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7034	return rc;
7035	}
7036
7037
7038	/**
7039	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7040	*/
7041	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7042	{
7043	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7044	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7045	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7046	AssertRCReturn(rc, rc);
7047
7048	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7049	if (RT_LIKELY(rc == VINF_SUCCESS))
7050	{
7051	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7052	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7053	AssertRCReturn(rc, rc);
7054	Assert(pVmxTransient->cbInstr == 3);
7055
7056	pMixedCtx->rip += pVmxTransient->cbInstr;
7057	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7058
7059	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7060	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7061	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7062	}
7063	else
7064	{
7065	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7066	rc = VERR_EM_INTERPRETER;
7067	}
7068	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7069	return rc;
7070	}
7071
7072
7073	/**
7074	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7075	*/
7076	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7077	{
7078	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7079	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7080	rc \|= hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7081	AssertRCReturn(rc, rc);
7082
7083	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7084	if (RT_LIKELY(rc == VINF_SUCCESS))
7085	{
7086	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7087	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7088	AssertRCReturn(rc, rc);
7089	Assert(pVmxTransient->cbInstr == 2);
7090
7091	pMixedCtx->rip += pVmxTransient->cbInstr;
7092	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7093	}
7094	else
7095	{
7096	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7097	rc = VERR_EM_INTERPRETER;
7098	}
7099	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7100	return rc;
7101	}
7102
7103
7104	/**
7105	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7106	*/
7107	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7108	{
7109	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7110	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7111	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7112	AssertRCReturn(rc, rc);
7113
7114	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7115	rc = VBOXSTRICTRC_VAL(rc2);
7116	if (RT_LIKELY(rc == VINF_SUCCESS))
7117	{
7118	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7119	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7120	AssertRCReturn(rc, rc);
7121
7122	pMixedCtx->rip += pVmxTransient->cbInstr;
7123	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7124	}
7125	else
7126	{
7127	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %RGv failed with %Rrc\n",
7128	pVmxTransient->uExitQualification, rc));
7129	rc = VERR_EM_INTERPRETER;
7130	}
7131	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7132	return rc;
7133	}
7134
7135
7136	/**
7137	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7138	*/
7139	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7140	{
7141	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7142	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7143	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7144	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7145	AssertRCReturn(rc, rc);
7146
7147	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7148	if (RT_LIKELY(rc == VINF_SUCCESS))
7149	{
7150	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7151	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7152	AssertRCReturn(rc, rc);
7153
7154	pMixedCtx->rip += pVmxTransient->cbInstr;
7155	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7156	}
7157	else
7158	{
7159	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7160	rc = VERR_EM_INTERPRETER;
7161	}
7162	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7163	return rc;
7164	}
7165
7166
7167	/**
7168	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7169	*/
7170	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7171	{
7172	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7173	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7174	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7175	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7176	AssertRCReturn(rc, rc);
7177
7178	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7179	rc = VBOXSTRICTRC_VAL(rc2);
7180	if (RT_LIKELY( rc == VINF_SUCCESS
7181	\|\| rc == VINF_EM_HALT))
7182	{
7183	int rc3 = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7184	rc3 \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7185	AssertRCReturn(rc3, rc3);
7186
7187	pMixedCtx->rip += pVmxTransient->cbInstr;
7188	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7189
7190	if ( rc == VINF_EM_HALT
7191	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7192	{
7193	rc = VINF_SUCCESS;
7194	}
7195	}
7196	else
7197	{
7198	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7199	rc = VERR_EM_INTERPRETER;
7200	}
7201	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7202	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7203	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7204	return rc;
7205	}
7206
7207
7208	/**
7209	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7210	*/
7211	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7212	{
7213	/*
7214	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7215	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7216	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7217	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7218	*/
7219	AssertMsgFailed(("Unexpected RSM VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7220	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7221	}
7222
7223
7224	/**
7225	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7226	*/
7227	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7228	{
7229	/*
7230	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7231	* root operation. If we get there there is something funny going on.
7232	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7233	*/
7234	AssertMsgFailed(("Unexpected SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7235	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7236	}
7237
7238
7239	/**
7240	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7241	*/
7242	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7243	{
7244	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7245	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7246	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7247	}
7248
7249
7250	/**
7251	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7252	*/
7253	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7254	{
7255	/*
7256	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7257	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7258	* See Intel spec. 25.3 "Other Causes of VM-exits".
7259	*/
7260	AssertMsgFailed(("Unexpected SIPI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7261	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7262	}
7263
7264
7265	/**
7266	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7267	* VM-exit.
7268	*/
7269	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7270	{
7271	/*
7272	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7273	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7274	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7275	*/
7276	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7277	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7278	}
7279
7280
7281	/**
7282	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7283	* VM-exit.
7284	*/
7285	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7286	{
7287	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7288	return VINF_EM_RESET;
7289	}
7290
7291
7292	/**
7293	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7294	*/
7295	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7296	{
7297	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7298	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7299	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7300	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7301	AssertRCReturn(rc, rc);
7302
7303	pMixedCtx->rip++;
7304	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7305	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7306	rc = VINF_SUCCESS;
7307	else
7308	rc = VINF_EM_HALT;
7309
7310	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7311	return rc;
7312	}
7313
7314
7315	/**
7316	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7317	*/
7318	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7319	{
7320	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7321	return hmR0VmxInjectXcptUD(pVM, pVCpu, pMixedCtx);
7322	}
7323
7324
7325	/**
7326	* VM-exit handler for expiry of the VMX preemption timer.
7327	*/
7328	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7329	{
7330	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7331
7332	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7333	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7334	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7335
7336	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7337	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7338	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7339	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7340	}
7341
7342
7343	/**
7344	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7345	*/
7346	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7347	{
7348	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7349	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7350	/** @todo check if XSETBV is supported by the recompiler. */
7351	return VERR_EM_INTERPRETER;
7352	}
7353
7354
7355	/**
7356	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7357	*/
7358	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7359	{
7360	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7361	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7362	/** @todo implement EMInterpretInvpcid() */
7363	return VERR_EM_INTERPRETER;
7364	}
7365
7366
7367	/**
7368	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7369	* Error VM-exit.
7370	*/
7371	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7372	{
7373	uint32_t uIntrState;
7374	RTHCUINTREG uHCReg;
7375	uint64_t u64Val;
7376
7377	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7378	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7379	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7380	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7381	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7382	AssertRCReturn(rc, rc);
7383
7384	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7385	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7386	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7387	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7388
7389	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7390	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7391	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7392	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7393	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7394	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7395	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7396	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7397	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7398	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7399	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7400	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7401
7402	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7403
7404	return VERR_VMX_INVALID_GUEST_STATE;
7405	}
7406
7407
7408	/**
7409	* VM-exit handler for VM-entry failure due to an MSR-load
7410	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7411	*/
7412	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7413	{
7414	AssertMsgFailed(("Unexpected MSR-load exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7415	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7416	}
7417
7418
7419	/**
7420	* VM-exit handler for VM-entry failure due to a machine-check event
7421	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7422	*/
7423	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7424	{
7425	AssertMsgFailed(("Unexpected machine-check event exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7426	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7427	}
7428
7429
7430	/**
7431	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7432	* theory.
7433	*/
7434	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7435	{
7436	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason,
7437	pVM, pVCpu, pMixedCtx));
7438	return VERR_VMX_UNDEFINED_EXIT_CODE;
7439	}
7440
7441
7442	/**
7443	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7444	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7445	* Conditional VM-exit.
7446	*/
7447	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7448	{
7449	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7450	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7451	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7452	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7453	return VERR_EM_INTERPRETER;
7454	AssertMsgFailed(("Unexpected XDTR access. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7455	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7456	}
7457
7458
7459	/**
7460	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7461	*/
7462	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7463	{
7464	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7465	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7466	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7467	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7468	return VERR_EM_INTERPRETER;
7469	AssertMsgFailed(("Unexpected RDRAND exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7470	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7471	}
7472
7473
7474	/**
7475	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7476	*/
7477	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7478	{
7479	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7480	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7481	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7482	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7483	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7484	AssertRCReturn(rc, rc);
7485
7486	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7487	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7488	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7489	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7490
7491	/* Update RIP and continue guest execution. */
7492	if (RT_LIKELY(rc == VINF_SUCCESS))
7493	{
7494	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7495	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7496	AssertRCReturn(rc, rc);
7497
7498	Assert(pVmxTransient->cbInstr == 2);
7499	pMixedCtx->rip += pVmxTransient->cbInstr;
7500	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7501	}
7502	return rc;
7503	}
7504
7505
7506	/**
7507	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7508	*/
7509	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7510	{
7511	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7512	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7513	AssertRCReturn(rc, rc);
7514	Assert(pVmxTransient->cbInstr == 2);
7515
7516	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7517	if ( pVM->hm.s.fTPRPatchingActive
7518	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7519	{
7520	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7521	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7522	{
7523	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7524	AssertRC(rc);
7525	}
7526
7527	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7528	AssertRCReturn(rc, rc);
7529	pMixedCtx->rip += pVmxTransient->cbInstr;
7530	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7531	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7532	return VINF_SUCCESS;
7533	}
7534
7535	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7536	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS))
7537	{
7538	switch (pMixedCtx->ecx)
7539	{
7540	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7541	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7542	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7543	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FS_BASE_MSR; break;
7544	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_GS_BASE_MSR; break;
7545	}
7546	}
7547	#ifdef DEBUG
7548	else
7549	{
7550	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7551	switch (pMixedCtx->ecx)
7552	{
7553	case MSR_IA32_SYSENTER_CS:
7554	case MSR_IA32_SYSENTER_EIP:
7555	case MSR_IA32_SYSENTER_ESP:
7556	case MSR_K8_FS_BASE:
7557	case MSR_K8_GS_BASE:
7558	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7559	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7560	case MSR_K8_LSTAR:
7561	case MSR_K6_STAR:
7562	case MSR_K8_SF_MASK:
7563	case MSR_K8_TSC_AUX:
7564	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7565	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7566	}
7567	}
7568	#endif
7569
7570	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7571	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7572	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7573	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7574	AssertRCReturn(rc, rc);
7575
7576	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7577	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7578	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7579
7580	/* Update guest-state and continue execution. */
7581	if (RT_LIKELY(rc == VINF_SUCCESS))
7582	{
7583	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7584	AssertRCReturn(rc, rc);
7585
7586	pMixedCtx->rip += pVmxTransient->cbInstr;
7587	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7588
7589	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7590	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7591	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7592	{
7593	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7594	}
7595	}
7596	return rc;
7597	}
7598
7599
7600	/**
7601	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7602	*/
7603	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7604	{
7605	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7606	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7607	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7608	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7609	return VERR_EM_INTERPRETER;
7610	AssertMsgFailed(("Unexpected PAUSE exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7611	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7612	}
7613
7614
7615	/**
7616	* VM-exit handler for when the TPR value is lowered below the specified
7617	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7618	*/
7619	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7620	{
7621	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7622	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7623
7624	/*
7625	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7626	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and
7627	* resume guest execution.
7628	*/
7629	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7630	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7631	return VINF_SUCCESS;
7632	}
7633
7634
7635	/**
7636	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7637	* VM-exit.
7638	*
7639	* @retval VINF_SUCCESS when guest execution can continue.
7640	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7641	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7642	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7643	* recompiler.
7644	*/
7645	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7646	{
7647	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7648	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7649	AssertRCReturn(rc, rc);
7650
7651	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7652	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7653	switch (uAccessType)
7654	{
7655	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7656	{
7657	#if 0
7658	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7659	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7660	#else
7661	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
7662	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7663	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7664	#endif
7665	AssertRCReturn(rc, rc);
7666
7667	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7668	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7669	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7670	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7671
7672	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7673	{
7674	case 0: /* CR0 */
7675	Log(("CR0 write rc=%d\n", rc));
7676	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7677	break;
7678	case 3: /* CR3 */
7679	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7680	Log(("CR3 write rc=%d\n", rc));
7681	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7682	break;
7683	case 4: /* CR4 */
7684	Log(("CR4 write rc=%d\n", rc));
7685	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7686	break;
7687	case 8: /* CR8 */
7688	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7689	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7690	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7691	break;
7692	default:
7693	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7694	break;
7695	}
7696
7697	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7698	break;
7699	}
7700
7701	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7702	{
7703	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7704	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7705	AssertRCReturn(rc, rc);
7706	Assert( !pVM->hm.s.fNestedPaging
7707	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7708	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7709
7710	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7711	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7712	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7713
7714	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7715	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7716	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7717	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7718	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7719	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7720	break;
7721	}
7722
7723	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7724	{
7725	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7726	AssertRCReturn(rc, rc);
7727	rc = EMInterpretCLTS(pVM, pVCpu);
7728	AssertRCReturn(rc, rc);
7729	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7730	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7731	Log(("CRX CLTS write rc=%d\n", rc));
7732	break;
7733	}
7734
7735	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7736	{
7737	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7738	AssertRCReturn(rc, rc);
7739	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7740	if (RT_LIKELY(rc == VINF_SUCCESS))
7741	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7742	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7743	Log(("CRX LMSW write rc=%d\n", rc));
7744	break;
7745	}
7746
7747	default:
7748	{
7749	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7750	return VERR_VMX_UNEXPECTED_EXCEPTION;
7751	}
7752	}
7753
7754	/* Validate possible error codes. */
7755	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3);
7756	if (RT_SUCCESS(rc))
7757	{
7758	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7759	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7760	AssertRCReturn(rc2, rc2);
7761	pMixedCtx->rip += pVmxTransient->cbInstr;
7762	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7763	}
7764
7765	return rc;
7766	}
7767
7768
7769	/**
7770	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7771	* VM-exit.
7772	*/
7773	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7774	{
7775	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7776
7777	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7778	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7779	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7780	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7781	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7782	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7783	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7784	AssertRCReturn(rc, rc);
7785
7786	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7787	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7788	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7789	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7790	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7791	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7792	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7793
7794	/* I/O operation lookup arrays. */
7795	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7796	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7797
7798	const uint32_t cbSize = s_aIOSize[uIOWidth];
7799	const uint32_t cbInstr = pVmxTransient->cbInstr;
7800	if (fIOString)
7801	{
7802	/* INS/OUTS - I/O String instruction. */
7803	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7804	/** @todo for now manually disassemble later optimize by getting the fields from
7805	* the VMCS. */
7806	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7807	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7808	* segment prefix info. */
7809	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7810	if (RT_SUCCESS(rc))
7811	{
7812	if (fIOWrite)
7813	{
7814	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7815	(DISCPUMODE)pDis->uAddrMode, cbSize);
7816	rc = VBOXSTRICTRC_VAL(rc2);
7817	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7818	}
7819	else
7820	{
7821	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7822	(DISCPUMODE)pDis->uAddrMode, cbSize);
7823	rc = VBOXSTRICTRC_VAL(rc2);
7824	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7825	}
7826	}
7827	else
7828	{
7829	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7830	rc = VINF_EM_RAW_EMULATE_INSTR;
7831	}
7832	}
7833	else
7834	{
7835	/* IN/OUT - I/O instruction. */
7836	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7837	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7838	if (fIOWrite)
7839	{
7840	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7841	rc = VBOXSTRICTRC_VAL(rc2);
7842	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7843	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7844	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7845	}
7846	else
7847	{
7848	uint32_t u32Result = 0;
7849	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7850	rc = VBOXSTRICTRC_VAL(rc2);
7851	if (IOM_SUCCESS(rc))
7852	{
7853	/* Save result of I/O IN instr. in AL/AX/EAX. */
7854	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7855	}
7856	else if (rc == VINF_IOM_R3_IOPORT_READ)
7857	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7858	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7859	}
7860	}
7861
7862	if (IOM_SUCCESS(rc))
7863	{
7864	pMixedCtx->rip += cbInstr;
7865	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7866	if (RT_LIKELY(rc == VINF_SUCCESS))
7867	{
7868	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx); /* For DR7. */
7869	AssertRCReturn(rc, rc);
7870
7871	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7872	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7873	{
7874	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7875	for (unsigned i = 0; i < 4; i++)
7876	{
7877	unsigned uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7878	if ( ( uIOPort >= pMixedCtx->dr[i]
7879	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7880	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7881	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7882	{
7883	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7884	uint64_t uDR6 = ASMGetDR6();
7885
7886	/* Clear all breakpoint status flags and set the one we just hit. */
7887	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
7888	uDR6 \|= (uint64_t)RT_BIT(i);
7889
7890	/*
7891	* Note: AMD64 Architecture Programmer's Manual 13.1:
7892	* Bits 15:13 of the DR6 register is never cleared by the processor and must
7893	* be cleared by software after the contents have been read.
7894	*/
7895	ASMSetDR6(uDR6);
7896
7897	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
7898	pMixedCtx->dr[7] &= ~X86_DR7_GD;
7899
7900	/* Paranoia. */
7901	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
7902	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
7903	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
7904
7905	/* Resync DR7 */
7906	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
7907	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7908
7909	/* Inject #DB and get on with guest execution. */
7910	rc = hmR0VmxInjectXcptDB(pVM, pVCpu, pMixedCtx);
7911	AssertRCReturn(rc, rc);
7912	break;
7913	}
7914	}
7915	}
7916	}
7917	}
7918
7919	#ifdef DEBUG
7920	if (rc == VINF_IOM_R3_IOPORT_READ)
7921	Assert(!fIOWrite);
7922	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
7923	Assert(fIOWrite);
7924	else
7925	{
7926	AssertMsg( RT_FAILURE(rc)
7927	\|\| rc == VINF_SUCCESS
7928	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
7929	\|\| rc == VINF_EM_RAW_GUEST_TRAP
7930	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
7931	}
7932	#endif
7933
7934	return rc;
7935	}
7936
7937
7938	/**
7939	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
7940	* VM-exit.
7941	*/
7942	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7943	{
7944	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7945
7946	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
7947	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7948	AssertRCReturn(rc, rc);
7949	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
7950	{
7951	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
7952	AssertRCReturn(rc, rc);
7953	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
7954	{
7955	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
7956	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7957	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7958	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7959	{
7960	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
7961	pVCpu->hm.s.Event.fPending = true;
7962	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
7963	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
7964	AssertRCReturn(rc, rc);
7965	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringErrorCode))
7966	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
7967	else
7968	pVCpu->hm.s.Event.u32ErrCode = 0;
7969	}
7970	}
7971	}
7972	/** @todo Emulate task switch someday, currently just going back to ring-3 for
7973	* emulation. */
7974	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7975	return VERR_EM_INTERPRETER;
7976	}
7977
7978
7979	/**
7980	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
7981	*/
7982	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7983	{
7984	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7985	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
7986	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
7987	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7988	AssertRCReturn(rc, rc);
7989	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
7990	return VINF_EM_DBG_STOP;
7991	}
7992
7993
7994	/**
7995	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
7996	*/
7997	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7998	{
7999	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8000	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8001
8002	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8003	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8004	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8005	return VINF_SUCCESS;
8006	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8007	return rc;
8008
8009	#if 0
8010	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8011	* just sync the whole thing. */
8012	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8013	#else
8014	/* Aggressive state sync. for now. */
8015	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8016	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8017	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8018	#endif
8019	AssertRCReturn(rc, rc);
8020
8021	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8022	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8023	switch (uAccessType)
8024	{
8025	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8026	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8027	{
8028	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8029	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8030	{
8031	AssertMsgFailed(("hmR0VmxExitApicAccess: can't touch TPR offset while using TPR shadowing.\n"));
8032	}
8033
8034	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8035	GCPhys &= PAGE_BASE_GC_MASK;
8036	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8037	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8038	CPUMCTX2CORE(pMixedCtx), GCPhys);
8039	rc = VBOXSTRICTRC_VAL(rc2);
8040	Log(("ApicAccess %RGp %#x rc=%d\n", GCPhys,
8041	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification), rc));
8042	if ( rc == VINF_SUCCESS
8043	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8044	\|\| rc == VERR_PAGE_NOT_PRESENT)
8045	{
8046	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8047	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8048	rc = VINF_SUCCESS;
8049	}
8050	break;
8051	}
8052
8053	default:
8054	rc = VINF_EM_RAW_EMULATE_INSTR;
8055	break;
8056	}
8057
8058	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8059	return rc;
8060	}
8061
8062
8063	/**
8064	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8065	* VM-exit.
8066	*/
8067	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8068	{
8069	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8070
8071	/* We should -not- get this VM-exit if the guest is debugging. */
8072	if (CPUMIsGuestDebugStateActive(pVCpu))
8073	{
8074	AssertMsgFailed(("Unexpected MOV DRx exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
8075	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8076	}
8077
8078	int rc = VERR_INTERNAL_ERROR_5;
8079	if ( !DBGFIsStepping(pVCpu)
8080	&& !CPUMIsHyperDebugStateActive(pVCpu))
8081	{
8082	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
8083
8084	/* Don't intercept MOV DRx. */
8085	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8086	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8087	AssertRCReturn(rc, rc);
8088
8089	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8090	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8091	AssertRC(rc);
8092	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8093
8094	#ifdef VBOX_WITH_STATISTICS
8095	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8096	AssertRCReturn(rc, rc);
8097	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8098	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8099	else
8100	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8101	#endif
8102	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8103	return VINF_SUCCESS;
8104	}
8105
8106	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
8107	* time and restore DRx registers afterwards */
8108	/*
8109	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8110	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8111	*/
8112	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8113	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8114	AssertRCReturn(rc, rc);
8115
8116	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8117	{
8118	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8119	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8120	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8121	if (RT_SUCCESS(rc))
8122	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8123	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8124	}
8125	else
8126	{
8127	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8128	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8129	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8130	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8131	}
8132
8133	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8134	if (RT_SUCCESS(rc))
8135	{
8136	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8137	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8138	AssertRCReturn(rc2, rc2);
8139	pMixedCtx->rip += pVmxTransient->cbInstr;
8140	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8141	}
8142	return rc;
8143	}
8144
8145
8146	/**
8147	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8148	* Conditional VM-exit.
8149	*/
8150	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8151	{
8152	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8153	Assert(pVM->hm.s.fNestedPaging);
8154
8155	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8156	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8157	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8158	return VINF_SUCCESS;
8159	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8160	return rc;
8161
8162	RTGCPHYS GCPhys = 0;
8163	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8164
8165	#if 0
8166	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
8167	#else
8168	/* Aggressive state sync. for now. */
8169	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8170	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8171	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8172	#endif
8173	AssertRCReturn(rc, rc);
8174
8175	/*
8176	* If we succeed, resume guest execution.
8177	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8178	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8179	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8180	* weird case. See @bugref{6043}.
8181	*/
8182	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8183	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8184	rc = VBOXSTRICTRC_VAL(rc2);
8185	if ( rc == VINF_SUCCESS
8186	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8187	\|\| rc == VERR_PAGE_NOT_PRESENT)
8188	{
8189	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8190	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8191	return VINF_SUCCESS;
8192	}
8193	return rc;
8194	}
8195
8196
8197	/**
8198	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8199	* VM-exit.
8200	*/
8201	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8202	{
8203	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8204	Assert(pVM->hm.s.fNestedPaging);
8205
8206	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8207	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8208	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8209	return VINF_SUCCESS;
8210	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8211	return rc;
8212
8213	RTGCPHYS GCPhys = 0;
8214	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8215	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8216	#if 0
8217	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
8218	#else
8219	/* Aggressive state sync. for now. */
8220	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8221	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8222	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8223	#endif
8224	AssertRCReturn(rc, rc);
8225
8226	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8227	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8228
8229	RTGCUINT uErrorCode = 0;
8230	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8231	uErrorCode \|= X86_TRAP_PF_ID;
8232	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8233	uErrorCode \|= X86_TRAP_PF_RW;
8234	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8235	uErrorCode \|= X86_TRAP_PF_P;
8236
8237	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8238	TRPMSetErrorCode(pVCpu, uErrorCode);
8239	TRPMSetFaultAddress(pVCpu, GCPhys);
8240
8241	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8242	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8243
8244	/* Handle the pagefault trap for the nested shadow table. */
8245	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8246	TRPMResetTrap(pVCpu);
8247
8248	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8249	if ( rc == VINF_SUCCESS
8250	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8251	\|\| rc == VERR_PAGE_NOT_PRESENT)
8252	{
8253	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8254	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8255	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8256	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8257	return VINF_SUCCESS;
8258	}
8259
8260	Log(("EPT return to ring-3 rc=%d\n"));
8261	return rc;
8262	}
8263
8264
8265	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8266	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8267	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8268	/**
8269	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8270	*/
8271	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8272	{
8273	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8274
8275	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
8276	AssertRCReturn(rc, rc);
8277	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8278
8279	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8280	{
8281	/* Old-style FPU error reporting needs some extra work. */
8282	/** @todo don't fall back to the recompiler, but do it manually. */
8283	return VERR_EM_INTERPRETER;
8284	}
8285	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8286	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8287	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8288	AssertRCReturn(rc, rc);
8289	return rc;
8290	}
8291
8292
8293	/**
8294	* VM-exit exception handler for #BP (Breakpoint exception).
8295	*/
8296	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8297	{
8298	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8299
8300	/** @todo Try optimize this by not saving the entire guest state unless
8301	* really needed. */
8302	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8303	AssertRCReturn(rc, rc);
8304	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8305
8306	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8307	if (rc == VINF_EM_RAW_GUEST_TRAP)
8308	{
8309	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8310	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8311	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8312	AssertRCReturn(rc, rc);
8313
8314	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8315	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8316	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8317	AssertRCReturn(rc, rc);
8318	}
8319
8320	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8321	return rc;
8322	}
8323
8324
8325	/**
8326	* VM-exit exception handler for #DB (Debug exception).
8327	*/
8328	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8329	{
8330	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8331
8332	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8333	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8334	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8335	AssertRCReturn(rc, rc);
8336
8337	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8338	uint64_t uDR6 = X86_DR6_INIT_VAL;
8339	uDR6 \|= (pVmxTransient->uExitQualification
8340	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8341	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8342	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8343	if (rc == VINF_EM_RAW_GUEST_TRAP)
8344	{
8345	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8346	pMixedCtx->dr[6] = uDR6;
8347
8348	if (CPUMIsGuestDebugStateActive(pVCpu))
8349	ASMSetDR6(pMixedCtx->dr[6]);
8350
8351	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8352	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8353
8354	/* Paranoia. */
8355	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8356	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8357	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8358
8359	/* Resync DR7. */
8360	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8361
8362	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8363	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8364	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8365	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8366	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8367	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8368	AssertRCReturn(rc,rc);
8369	return rc;
8370	}
8371	/* Return to ring 3 to deal with the debug exit code. */
8372	return rc;
8373	}
8374
8375
8376	/**
8377	* VM-exit exception handler for #NM (Device-not-available exception: floating
8378	* point exception).
8379	*/
8380	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8381	{
8382	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8383
8384	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8385	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8386	#endif
8387
8388	/* We require CR0 and EFER. EFER is always up-to-date. */
8389	int rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8390	AssertRCReturn(rc, rc);
8391
8392	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8393	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8394	if (rc == VINF_SUCCESS)
8395	{
8396	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8397	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8398	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8399	return VINF_SUCCESS;
8400	}
8401
8402	/* Forward #NM to the guest. */
8403	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8404	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8405	AssertRCReturn(rc, rc);
8406	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8407	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8408	pVmxTransient->cbInstr, 0 /* error code */);
8409	AssertRCReturn(rc, rc);
8410	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8411	return rc;
8412	}
8413
8414
8415	/**
8416	* VM-exit exception handler for #GP (General-protection exception).
8417	*
8418	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8419	*/
8420	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8421	{
8422	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8423
8424	int rc = VERR_INTERNAL_ERROR_5;
8425
8426	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8427	{
8428	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8429	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8430	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8431	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8432	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8433	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8434	Log(("#GP Gst: RIP %#RX64\n", pMixedCtx->rip));
8435	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8436	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8437	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8438	AssertRCReturn(rc, rc);
8439	return rc;
8440	#else
8441	/* We don't intercept #GP. */
8442	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8443	return VERR_VMX_UNEXPECTED_EXCEPTION;
8444	#endif
8445	}
8446
8447	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8448	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
8449
8450	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8451	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8452	AssertRCReturn(rc, rc);
8453
8454	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8455	unsigned int cbOp = 0;
8456	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8457	if (RT_SUCCESS(rc))
8458	{
8459	rc = VINF_SUCCESS;
8460	Assert(cbOp == pDis->cbInstr);
8461	Log2(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8462	switch (pDis->pCurInstr->uOpcode)
8463	{
8464	case OP_CLI:
8465	pMixedCtx->eflags.Bits.u1IF = 0;
8466	pMixedCtx->rip += pDis->cbInstr;
8467	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8468	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8469	break;
8470
8471	case OP_STI:
8472	pMixedCtx->eflags.Bits.u1IF = 1;
8473	pMixedCtx->rip += pDis->cbInstr;
8474	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8475	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8476	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8477	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8478	break;
8479
8480	case OP_HLT:
8481	rc = VINF_EM_HALT;
8482	pMixedCtx->rip += pDis->cbInstr;
8483	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8484	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8485	break;
8486
8487	case OP_POPF:
8488	{
8489	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8490	uint32_t cbParm = 0;
8491	uint32_t uMask = 0;
8492	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8493	{
8494	cbParm = 4;
8495	uMask = 0xffffffff;
8496	}
8497	else
8498	{
8499	cbParm = 2;
8500	uMask = 0xffff;
8501	}
8502
8503	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8504	RTGCPTR GCPtrStack = 0;
8505	X86EFLAGS uEflags;
8506	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8507	&GCPtrStack);
8508	if (RT_SUCCESS(rc))
8509	{
8510	Assert(sizeof(uEflags.u32) >= cbParm);
8511	uEflags.u32 = 0;
8512	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8513	}
8514	if (RT_FAILURE(rc))
8515	{
8516	rc = VERR_EM_INTERPRETER;
8517	break;
8518	}
8519	Log(("POPF %x -> %RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8520	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8521	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8522	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8523	pMixedCtx->eflags.Bits.u1RF = 0;
8524	pMixedCtx->esp += cbParm;
8525	pMixedCtx->esp &= uMask;
8526	pMixedCtx->rip += pDis->cbInstr;
8527	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8528	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8529	break;
8530	}
8531
8532	case OP_PUSHF:
8533	{
8534	uint32_t cbParm = 0;
8535	uint32_t uMask = 0;
8536	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8537	{
8538	cbParm = 4;
8539	uMask = 0xffffffff;
8540	}
8541	else
8542	{
8543	cbParm = 2;
8544	uMask = 0xffff;
8545	}
8546
8547	/* Get the stack pointer & push the contents of eflags onto the stack. */
8548	RTGCPTR GCPtrStack = 0;
8549	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8550	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8551	if (RT_FAILURE(rc))
8552	{
8553	rc = VERR_EM_INTERPRETER;
8554	break;
8555	}
8556	X86EFLAGS uEflags;
8557	uEflags = pMixedCtx->eflags;
8558	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8559	uEflags.Bits.u1RF = 0;
8560	uEflags.Bits.u1VM = 0;
8561
8562	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8563	if (RT_FAILURE(rc))
8564	{
8565	rc = VERR_EM_INTERPRETER;
8566	break;
8567	}
8568	Log(("PUSHF %x -> %RGv\n", uEflags.u, GCPtrStack));
8569	pMixedCtx->esp -= cbParm;
8570	pMixedCtx->esp &= uMask;
8571	pMixedCtx->rip += pDis->cbInstr;
8572	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8573	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8574	break;
8575	}
8576
8577	case OP_IRET:
8578	{
8579	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8580	* instruction reference. */
8581	RTGCPTR GCPtrStack = 0;
8582	uint32_t uMask = 0xffff;
8583	uint16_t aIretFrame[3];
8584	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8585	{
8586	rc = VERR_EM_INTERPRETER;
8587	break;
8588	}
8589	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8590	&GCPtrStack);
8591	if (RT_SUCCESS(rc))
8592	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8593	if (RT_FAILURE(rc))
8594	{
8595	rc = VERR_EM_INTERPRETER;
8596	break;
8597	}
8598	pMixedCtx->eip = 0;
8599	pMixedCtx->ip = aIretFrame[0];
8600	pMixedCtx->cs.Sel = aIretFrame[1];
8601	pMixedCtx->cs.ValidSel = aIretFrame[1];
8602	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8603	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8604	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8605	pMixedCtx->sp += sizeof(aIretFrame);
8606	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8607	\| HM_CHANGED_GUEST_RFLAGS;
8608	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8609	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8610	break;
8611	}
8612
8613	case OP_INT:
8614	{
8615	uint16_t uVector = pDis->Param1.uValue & 0xff;
8616	rc = hmR0VmxInjectIntN(pVM, pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8617	AssertRCReturn(rc, rc);
8618	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8619	break;
8620	}
8621
8622	case OP_INTO:
8623	{
8624	if (pMixedCtx->eflags.Bits.u1OF)
8625	{
8626	rc = hmR0VmxInjectXcptOF(pVM, pVCpu, pMixedCtx, pDis->cbInstr);
8627	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8628	}
8629	break;
8630	}
8631
8632	default:
8633	{
8634	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8635	EMCODETYPE_SUPERVISOR);
8636	rc = VBOXSTRICTRC_VAL(rc2);
8637	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8638	Log2(("#GP rc=%Rrc\n", rc));
8639	break;
8640	}
8641	}
8642	}
8643	else
8644	rc = VERR_EM_INTERPRETER;
8645
8646	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT
8647	\|\| rc == VINF_EM_RESET /* injection caused triple fault */,
8648	("#GP Unexpected rc=%Rrc\n", rc));
8649	return rc;
8650	}
8651
8652
8653	/**
8654	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8655	* the exception reported in the VMX transient structure back into the VM.
8656	*
8657	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8658	* VMX transient structure to be up-to-date.
8659	*/
8660	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8661	{
8662	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8663
8664	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8665	hmR0VmxCheckExitDueToEventDelivery(). */
8666	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8667	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8668	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8669	AssertRCReturn(rc, rc);
8670	return rc;
8671	}
8672
8673
8674	/**
8675	* VM-exit exception handler for #PF (Page-fault exception).
8676	*/
8677	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8678	{
8679	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8680
8681	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8682	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8683	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8684	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8685	AssertRCReturn(rc, rc);
8686
8687	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8688	if (pVM->hm.s.fNestedPaging)
8689	{
8690	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8691	{
8692	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8693	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8694	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8695	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8696	AssertRCReturn(rc, rc);
8697	}
8698	else
8699	{
8700	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8701	Assert(!pVCpu->hm.s.Event.fPending);
8702	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8703	}
8704	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8705	return rc;
8706	}
8707	#else
8708	Assert(!pVM->hm.s.fNestedPaging);
8709	#endif
8710
8711	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8712	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8713	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8714	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8715	AssertRCReturn(rc, rc);
8716	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8717	if ( pVM->hm.s.fTRPPatchingAllowed
8718	&& pVM->hm.s.pGuestPatchMem
8719	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8720	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8721	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8722	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8723	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8724	{
8725	RTGCPHYS GCPhys;
8726	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8727	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8728	if ( rc == VINF_SUCCESS
8729	&& GCPhys == GCPhysApicBase)
8730	{
8731	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8732	AssertRCReturn(rc, rc);
8733
8734	/* Only attempt to patch the instruction once. */
8735	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8736	if (!pPatch)
8737	return VINF_EM_HM_PATCH_TPR_INSTR;
8738	}
8739	}
8740	#endif
8741
8742	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8743	TRPMSetFaultAddress(pVCpu, pVmxTransient->uExitQualification);
8744	TRPMSetErrorCode(pVCpu, pVmxTransient->uExitIntrErrorCode);
8745
8746	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8747	AssertRCReturn(rc, rc);
8748
8749	/* Forward it to the trap handler first. */
8750	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8751	(RTGCPTR)pVmxTransient->uExitQualification);
8752
8753	Log(("#PF: cr2=%RGv cs:rip=%04x:%RGv errorcode %#RX32 rc=%d\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8754	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, rc));
8755
8756	if (rc == VINF_SUCCESS)
8757	{
8758	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8759	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8760	* memory? We don't update the whole state here... */
8761	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8762	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8763	TRPMResetTrap(pVCpu);
8764	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8765	return rc;
8766	}
8767	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8768	{
8769	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8770	{
8771	/* It's a guest page fault and needs to be reflected to the guest. */
8772	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8773	TRPMResetTrap(pVCpu);
8774	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8775	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8776	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8777	pVmxTransient->cbInstr, uGstErrorCode);
8778	AssertRCReturn(rc, rc);
8779	}
8780	else
8781	{
8782	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8783	Assert(!pVCpu->hm.s.Event.fPending);
8784	TRPMResetTrap(pVCpu);
8785	Log(("#PF: Injecting #DF\n"));
8786	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8787	}
8788	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8789	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8790	return rc;
8791	}
8792
8793	TRPMResetTrap(pVCpu);
8794	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8795	return rc;
8796	}
8797

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45494

Download in other formats: